[1/18] argv_ref speedup: replace array with accessor struct

From 14b2ffbb82eb22cd36b1703a5501ee87ea51c45c Mon Sep 17 00:00:00 2001
From: Eric Blake <[email protected]>
Date: Sat, 24 Nov 2007 16:23:47 -0700
Subject: [PATCH] Stage 1: convert m4_symbol_value** into new object.

* ltdl/m4/gnulib-cache.m4: Import flexmember module.
* m4/m4module.h (m4_macro_args): New type, will be opaque to
modules later.
(m4_builtin_func, M4BUILTIN, M4BUILTIN_HANDLER, m4_dump_args)
(m4_macro_call): Alter signature to use m4_macro_args object.
(M4ARG): Redefine to access new struct.
* m4/m4private.h (M4_SYMBOL_COMP): New enumerator.
(struct m4_symbol_chain): New type.
(struct m4_symbol_value): Add chain alternative.
* m4/macro.c (collect_arguments): Build new struct.
(expand_macro, m4_macro_call, process_macro, trace_pre)
(trace_post): Adjust implementation to use it.
* m4/utility.c (m4_dump_args): Likewise.
* modules/gnu.c (builtin, indir): Likewise.
* modules/format.c (format): Likewise.
* modules/m4.h (m4_dump_symbols_func): Likewise.
* modules/m4.c (m4_dump_symbols, define, pushdef, defn, ifelse)
(shift, include, errprint, m4wrap): Likewise.
* modules/evalparse.c (m4_evaluate): Likewise.

Signed-off-by: Eric Blake <[email protected]>
---
 ChangeLog               |   21 ++++++++++
 ltdl/m4/gnulib-cache.m4 |    4 +-
 m4/m4module.h           |   72 ++++++++++++++++++++++------------
 m4/m4private.h          |   30 +++++++++++---
 m4/macro.c              |   97 ++++++++++++++++++++++++----------------------
 m4/utility.c            |   16 +++++---
 modules/evalparse.c     |    2 +-
 modules/format.c        |   52 ++++++++++++-------------
 modules/gnu.c           |   49 +++++++++++++++++------
 modules/m4.c            |   42 +++++++++++----------
 modules/m4.h            |    2 +-
 11 files changed, 238 insertions(+), 149 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f71a00f..2fbe4ee 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,26 @@
 2007-11-24  Eric Blake  <[email protected]>
 
+	Stage 1: convert m4_symbol_value** into new object.
+	* ltdl/m4/gnulib-cache.m4: Import flexmember module.
+	* m4/m4module.h (m4_macro_args): New type, will be opaque to
+	modules later.
+	(m4_builtin_func, M4BUILTIN, M4BUILTIN_HANDLER, m4_dump_args)
+	(m4_macro_call): Alter signature to use m4_macro_args object.
+	(M4ARG): Redefine to access new struct.
+	* m4/m4private.h (M4_SYMBOL_COMP): New enumerator.
+	(struct m4_symbol_chain): New type.
+	(struct m4_symbol_value): Add chain alternative.
+	* m4/macro.c (collect_arguments): Build new struct.
+	(expand_macro, m4_macro_call, process_macro, trace_pre)
+	(trace_post): Adjust implementation to use it.
+	* m4/utility.c (m4_dump_args): Likewise.
+	* modules/gnu.c (builtin, indir): Likewise.
+	* modules/format.c (format): Likewise.
+	* modules/m4.h (m4_dump_symbols_func): Likewise.
+	* modules/m4.c (m4_dump_symbols, define, pushdef, defn, ifelse)
+	(shift, include, errprint, m4wrap): Likewise.
+	* modules/evalparse.c (m4_evaluate): Likewise.
+
 	Pass only macro name to utility functions.
 	* m4/m4module.h (m4_bad_argc, m4_numeric_arg): Adjust signature.
 	* m4/utility.c (m4_bad_argc, m4_numeric_arg): Adjust
diff --git a/ltdl/m4/gnulib-cache.m4 b/ltdl/m4/gnulib-cache.m4
index 9c17a78..1eefbd5 100644
--- a/ltdl/m4/gnulib-cache.m4
+++ b/ltdl/m4/gnulib-cache.m4
@@ -15,11 +15,11 @@
 
 
 # Specification in the form of a command-line invocation:
-#   gnulib-tool --import --dir=. --local-dir=local --lib=libgnu --source-base=gnu --m4-base=ltdl/m4 --doc-base=doc --aux-dir=build-aux --with-tests --libtool --macro-prefix=M4 assert autobuild avltree-oset binary-io clean-temp cloexec close-stream closein config-h configmake dirname error exit fdl fflush filenamecat fopen-safer free fseeko gendocs gettext gnupload gpl-3.0 mkstemp obstack progname regex regexprops-generic sprintf-posix stdbool stdlib-safer strnlen strtol tempname unlocked-io verror xalloc xalloc-die xprintf-posix xstrndup xvasprintf-posix
+#   gnulib-tool --import --dir=. --local-dir=local --lib=libgnu --source-base=gnu --m4-base=ltdl/m4 --doc-base=doc --aux-dir=build-aux --with-tests --libtool --macro-prefix=M4 assert autobuild avltree-oset binary-io clean-temp cloexec close-stream closein config-h configmake dirname error exit fdl fflush filenamecat flexmember fopen-safer free fseeko gendocs gettext gnupload gpl-3.0 mkstemp obstack progname regex regexprops-generic sprintf-posix stdbool stdlib-safer strnlen strtol tempname unlocked-io verror xalloc xalloc-die xprintf-posix xstrndup xvasprintf-posix
 
 # Specification in the form of a few gnulib-tool.m4 macro invocations:
 gl_LOCAL_DIR([local])
-gl_MODULES([assert autobuild avltree-oset binary-io clean-temp cloexec close-stream closein config-h configmake dirname error exit fdl fflush filenamecat fopen-safer free fseeko gendocs gettext gnupload gpl-3.0 mkstemp obstack progname regex regexprops-generic sprintf-posix stdbool stdlib-safer strnlen strtol tempname unlocked-io verror xalloc xalloc-die xprintf-posix xstrndup xvasprintf-posix])
+gl_MODULES([assert autobuild avltree-oset binary-io clean-temp cloexec close-stream closein config-h configmake dirname error exit fdl fflush filenamecat flexmember fopen-safer free fseeko gendocs gettext gnupload gpl-3.0 mkstemp obstack progname regex regexprops-generic sprintf-posix stdbool stdlib-safer strnlen strtol tempname unlocked-io verror xalloc xalloc-die xprintf-posix xstrndup xvasprintf-posix])
 gl_AVOID([])
 gl_SOURCE_BASE([gnu])
 gl_M4_BASE([ltdl/m4])
diff --git a/m4/m4module.h b/m4/m4module.h
index 01514db..3f60cba 100644
--- a/m4/m4module.h
+++ b/m4/m4module.h
@@ -37,10 +37,11 @@ typedef struct m4_macro		m4_macro;
 typedef struct m4_symbol_value	m4_symbol_value;
 typedef struct m4_input_block	m4_input_block;
 typedef struct m4_module	m4_module;
+typedef struct m4_macro_args	m4_macro_args;
 
 typedef struct obstack		m4_obstack;
 
-typedef void   m4_builtin_func  (m4 *, m4_obstack *, int, m4_symbol_value **);
+typedef void   m4_builtin_func  (m4 *, m4_obstack *, int, m4_macro_args *);
 
 /* The value of m4_builtin flags is built from these:  */
 enum {
@@ -75,38 +76,58 @@ struct m4_macro
   const char *value;
 };
 
+/* FIXME - make this struct opaque.  */
+struct m4_macro_args
+{
+  /* One more than the highest actual argument.  May be larger than
+     arraylen since the array can refer to multiple arguments via a
+     single $@ reference.  */
+  unsigned int argc;
+  /* False unless the macro expansion refers to $@; determines whether
+     this object can be freed at end of macro expansion or must wait
+     until all references have been rescanned.  */
+  bool inuse;
+  const char *argv0; /* The macro name being expanded.  */
+  size_t arraylen; /* True length of allocated elements in array.  */
+  /* Used as a variable-length array, storing information about each
+     argument.  */
+  m4_symbol_value *array[FLEXIBLE_ARRAY_MEMBER];
+};
+
 
-#define M4BUILTIN(name)					\
-  static void CONC(builtin_, name)				\
-   (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv);
+#define M4BUILTIN(name)							\
+  static void CONC (builtin_, name)					\
+   (m4 *context, m4_obstack *obs, int argc, m4_macro_args *argv);
 
-#define M4BUILTIN_HANDLER(name)				\
-  static void CONC(builtin_, name)				\
-   (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv)
+#define M4BUILTIN_HANDLER(name)						\
+  static void CONC (builtin_, name)					\
+   (m4 *context, m4_obstack *obs, int argc, m4_macro_args *argv)
 
-#define M4INIT_HANDLER(name)					\
-  void CONC(name, CONC(_LTX_, m4_init_module))			\
-	(m4 *context, m4_module *module, m4_obstack *obs);	\
-  void CONC(name, CONC(_LTX_, m4_init_module))			\
+#define M4INIT_HANDLER(name)						\
+  void CONC (name, CONC (_LTX_, m4_init_module))			\
+       (m4 *context, m4_module *module, m4_obstack *obs);		\
+  void CONC (name, CONC (_LTX_, m4_init_module))			\
 	(m4 *context, m4_module *module, m4_obstack *obs)
 
-#define M4FINISH_HANDLER(name)					\
-  void CONC(name, CONC(_LTX_, m4_finish_module))		\
-	(m4 *context, m4_module *module, m4_obstack *obs);	\
-  void CONC(name, CONC(_LTX_, m4_finish_module))		\
+#define M4FINISH_HANDLER(name)						\
+  void CONC (name, CONC (_LTX_, m4_finish_module))			\
+       (m4 *context, m4_module *module, m4_obstack *obs);		\
+  void CONC (name, CONC (_LTX_, m4_finish_module))			\
 	(m4 *context, m4_module *module, m4_obstack *obs)
 
-#define M4_MODULE_IMPORT(M, S)					\
-  CONC(S, _func) *S = (CONC(S, _func) *)			\
-	m4_module_import (context, STR(M), STR(S), obs)
+#define M4_MODULE_IMPORT(M, S)						\
+  CONC (S, _func) *S = (CONC (S, _func) *)				\
+	m4_module_import (context, STR (M), STR (S), obs)
 
-#define M4ARG(i)	(argc > (i) ? m4_get_symbol_value_text (argv[i]) : "")
+#define M4ARG(i)							\
+  ((i) == 0 ? argv->argv0						\
+   : argv->argc > (i) ? m4_get_symbol_value_text (argv->array[(i) - 1]) : "")
 
 extern bool	m4_bad_argc	   (m4 *, int, const char *,
 				    unsigned int, unsigned int, bool);
 extern bool	m4_numeric_arg	   (m4 *, const char *, const char *, int *);
 extern void	m4_dump_args	   (m4 *, m4_obstack *, int,
-				    m4_symbol_value **, const char *, bool);
+				    m4_macro_args *, const char *, bool);
 extern bool	m4_parse_truth_arg (m4 *, const char *, const char *, bool);
 
 /* Error handling.  */
@@ -164,14 +185,14 @@ extern void		m4_delete	(m4 *);
 
 
 #define M4FIELD(type, base, field)					\
-	extern type CONC(m4_get_, base) (m4 *context);			\
-	extern type CONC(m4_set_, base) (m4 *context, type value);
+	extern type CONC (m4_get_, base) (m4 *context);			\
+	extern type CONC (m4_set_, base) (m4 *context, type value);
 m4_context_field_table
 #undef M4FIELD
 
 #define M4OPT_BIT(bit, base)						\
-	extern bool CONC(m4_get_, base) (m4 *context);		\
-	extern bool CONC(m4_set_, base) (m4 *context, bool value);
+  extern bool CONC (m4_get_, base) (m4 *context);			\
+  extern bool CONC (m4_set_, base) (m4 *context, bool value);
 m4_context_opt_bit_table
 #undef M4OPT_BIT
 
@@ -287,8 +308,7 @@ extern const m4_builtin	*m4_builtin_find_by_func (m4_module *,
 
 extern void	   m4_macro_expand_input (m4 *);
 extern void	   m4_macro_call	 (m4 *, m4_symbol_value *,
-					  m4_obstack *, int,
-					  m4_symbol_value **);
+					  m4_obstack *, int, m4_macro_args *);
 
 
 
diff --git a/m4/m4private.h b/m4/m4private.h
index b067254..63a9bc4 100644
--- a/m4/m4private.h
+++ b/m4/m4private.h
@@ -30,10 +30,11 @@
 typedef struct m4__search_path_info m4__search_path_info;
 
 typedef enum {
-  M4_SYMBOL_VOID, /* Traced but undefined.  */
-  M4_SYMBOL_TEXT, /* Plain text.  */
-  M4_SYMBOL_FUNC, /* Builtin function.  */
-  M4_SYMBOL_PLACEHOLDER /* Placeholder for unknown builtin during -R.  */
+  M4_SYMBOL_VOID,		/* Traced but undefined, u is invalid.  */
+  M4_SYMBOL_TEXT,		/* Plain text, u.u_t is valid.  */
+  M4_SYMBOL_FUNC,		/* Builtin function, u.func is valid.  */
+  M4_SYMBOL_PLACEHOLDER,	/* Placeholder for unknown builtin from -R.  */
+  M4_SYMBOL_COMP		/* Composite symbol, u.chain is valid.  */
 } m4__symbol_type;
 
 #define BIT_TEST(flags, bit)	(((flags) & (bit)) == (bit))
@@ -171,13 +172,27 @@ extern m4_module *  m4__module_find (const char *name);
 
 /* --- SYMBOL TABLE MANAGEMENT --- */
 
+typedef struct m4_symbol_chain m4_symbol_chain;
+
 struct m4_symbol
 {
   bool		traced;
   m4_symbol_value *	value;
 };
 
-struct m4_symbol_value {
+/* Composite symbols are built of a linked list of chain objects.  */
+struct m4_symbol_chain
+{
+  m4_symbol_chain *next;/* Pointer to next link of chain.  */
+  char *str;		/* NUL-terminated string if text, else NULL.  */
+  m4_macro_args *argv;	/* Reference to earlier $@.  */
+  unsigned int index;	/* Index within argv to start reading from.  */
+};
+
+/* A symbol value is used both for values associated with a macro
+   name, and for arguments to a macro invocation.  */
+struct m4_symbol_value
+{
   m4_symbol_value *	next;
   m4_module *		module;
   unsigned int		flags;
@@ -189,8 +204,9 @@ struct m4_symbol_value {
 
   m4__symbol_type	type;
   union {
-    const char *	text; /* Valid when type is TEXT, PLACEHOLDER.  */
-    const m4_builtin *	builtin; /* Valid when type is FUNC.  */
+    const char *	text;	/* Valid when type is TEXT, PLACEHOLDER.  */
+    const m4_builtin *	builtin;/* Valid when type is FUNC.  */
+    m4_symbol_chain *	chain;	/* Valid when type is COMP.  */
   } u;
 };
 
diff --git a/m4/macro.c b/m4/macro.c
index dd93152..c4ba1af 100644
--- a/m4/macro.c
+++ b/m4/macro.c
@@ -29,22 +29,23 @@
 
 #include "intprops.h"
 
-static void    collect_arguments (m4 *, const char *, m4_symbol *,
-				  m4_obstack *, m4_obstack *);
+static m4_macro_args *collect_arguments (m4 *, const char *, m4_symbol *,
+					 m4_obstack *, unsigned int,
+					 m4_obstack *);
 static void    expand_macro      (m4 *, const char *, m4_symbol *);
 static void    expand_token      (m4 *, m4_obstack *, m4__token_type,
 				  m4_symbol_value *, int);
 static bool    expand_argument   (m4 *, m4_obstack *, m4_symbol_value *,
 				  const char *);
 static void    process_macro	 (m4 *, m4_symbol_value *, m4_obstack *, int,
-				  m4_symbol_value **);
+				  m4_macro_args *);
 
 static void    trace_prepre	 (m4 *, const char *, size_t,
 				  m4_symbol_value *);
 static void    trace_pre	 (m4 *, const char *, size_t, int,
-				  m4_symbol_value **);
+				  m4_macro_args *);
 static void    trace_post	 (m4 *, const char *, size_t, int,
-				  m4_symbol_value **, m4_input_block *, bool);
+				  m4_macro_args *, m4_input_block *, bool);
 
 static void    trace_format	 (m4 *, const char *, ...)
   M4_GNUC_PRINTF (2, 3);
@@ -126,11 +127,11 @@ expand_token (m4 *context, m4_obstack *obs,
 	  ++textp;
 
 	symbol = m4_symbol_lookup (M4SYMTAB, textp);
-	assert (! symbol || ! m4_is_symbol_void (symbol));
+	assert (!symbol || !m4_is_symbol_void (symbol));
 	if (symbol == NULL
 	    || (symbol->value->type == M4_SYMBOL_FUNC
 		&& BIT_TEST (SYMBOL_FLAGS (symbol), VALUE_BLIND_ARGS_BIT)
-		&& ! m4__next_token_is_open (context)))
+		&& !m4__next_token_is_open (context)))
 	  {
 	    m4_shipout_text (context, obs, text, strlen (text), line);
 	  }
@@ -248,7 +249,7 @@ expand_macro (m4 *context, const char *name, m4_symbol *symbol)
   char *argc_base = NULL;	/* Base of argc_stack on entry.  */
   unsigned int argc_size;	/* Size of argc_stack on entry.  */
   unsigned int argv_size;	/* Size of argv_stack on entry.  */
-  m4_symbol_value **argv;
+  m4_macro_args *argv;
   int argc;
   m4_obstack *expansion;
   m4_input_block *expanded;
@@ -297,15 +298,12 @@ recursion limit of %zu exceeded, use -L<N> to change it"),
   if (traced && m4_is_debug_bit (context, M4_DEBUG_TRACE_CALL))
     trace_prepre (context, name, my_call_id, value);
 
-  collect_arguments (context, name, symbol, &argv_stack, &argc_stack);
-
-  argc = ((obstack_object_size (&argv_stack) - argv_size)
-	  / sizeof (m4_symbol_value *));
-  argv = (m4_symbol_value **) ((char *) obstack_base (&argv_stack)
-			       + argv_size);
+  argv = collect_arguments (context, name, symbol, &argv_stack, argv_size,
+			    &argc_stack);
+  argc = argv->argc;
   /* Calling collect_arguments invalidated name, but we copied it as
      argv[0].  */
-  name = m4_get_symbol_value_text (argv[0]);
+  name = argv->argv0;
 
   loc_close_file = m4_get_current_file (context);
   loc_close_line = m4_get_current_line (context);
@@ -331,32 +329,39 @@ recursion limit of %zu exceeded, use -L<N> to change it"),
   if (BIT_TEST (VALUE_FLAGS (value), VALUE_DELETED_BIT))
     m4_symbol_value_delete (value);
 
+  /* TODO - pay attention to argv->inuse.  */
   if (0 < argc_size)
     obstack_regrow (&argc_stack, argc_base, argc_size);
   else
-    obstack_free (&argc_stack, argv[0]);
-  obstack_blank (&argv_stack, -argc * sizeof (m4_symbol_value *));
+    obstack_free (&argc_stack, (void *) name);
+  obstack_blank (&argv_stack, argv_size - obstack_object_size (&argv_stack));
 }
 
-/* Collect all the arguments to a call of the macro SYMBOL (called NAME).
-   The arguments are stored on the obstack ARGUMENTS and a table of pointers
-   to the arguments on the obstack ARGPTR.  */
-static void
+/* Collect all the arguments to a call of the macro SYMBOL (called
+   NAME).  The arguments are stored on the obstack ARGUMENTS and a
+   table of pointers to the arguments on the obstack ARGPTR.  ARGPTR
+   is an incomplete object, currently occupying ARGV_BASE bytes.
+   Return the object describing all of the macro arguments.  */
+static m4_macro_args *
 collect_arguments (m4 *context, const char *name, m4_symbol *symbol,
-		   m4_obstack *argptr, m4_obstack *arguments)
+		   m4_obstack *argptr, unsigned argv_base,
+		   m4_obstack *arguments)
 {
   m4_symbol_value token;
   m4_symbol_value *tokenp;
   bool more_args;
   bool groks_macro_args;
+  m4_macro_args args;
+  m4_macro_args *argv;
 
   groks_macro_args = BIT_TEST (SYMBOL_FLAGS (symbol), VALUE_MACRO_ARGS_BIT);
 
-  tokenp = (m4_symbol_value *) obstack_alloc (arguments, sizeof *tokenp);
-  m4_set_symbol_value_text (tokenp, (char *) obstack_copy0 (arguments, name,
-							    strlen (name)));
-  name = m4_get_symbol_value_text (tokenp);
-  obstack_ptr_grow (argptr, tokenp);
+  args.argc = 1;
+  args.inuse = false;
+  args.argv0 = (char *) obstack_copy0 (arguments, name, strlen (name));
+  args.arraylen = 0;
+  obstack_grow (argptr, &args, offsetof (m4_macro_args, array));
+  name = args.argv0;
 
   if (m4__next_token_is_open (context))
     {
@@ -373,9 +378,15 @@ collect_arguments (m4 *context, const char *name, m4_symbol *symbol,
 	  tokenp = (m4_symbol_value *) obstack_copy (arguments, &token,
 						     sizeof token);
 	  obstack_ptr_grow (argptr, tokenp);
+	  args.arraylen++;
+	  args.argc++;
 	}
       while (more_args);
     }
+  argv = (m4_macro_args *) ((char *) obstack_base (argptr) + argv_base);
+  argv->argc = args.argc;
+  argv->arraylen = args.arraylen;
+  return argv;
 }
 
 
@@ -387,30 +398,24 @@ collect_arguments (m4 *context, const char *name, m4_symbol *symbol,
    the obstack EXPANSION.  Macro tracing is also handled here.  */
 void
 m4_macro_call (m4 *context, m4_symbol_value *value, m4_obstack *expansion,
-	       int argc, m4_symbol_value **argv)
+	       int argc, m4_macro_args *argv)
 {
-  if (m4_bad_argc (context, argc, m4_get_symbol_value_text (argv[0]),
+  if (m4_bad_argc (context, argc, argv->argv0,
 		   VALUE_MIN_ARGS (value), VALUE_MAX_ARGS (value),
 		   BIT_TEST (VALUE_FLAGS (value),
 			     VALUE_SIDE_EFFECT_ARGS_BIT)))
     return;
   if (m4_is_symbol_value_text (value))
-    {
-      process_macro (context, value, expansion, argc, argv);
-    }
+    process_macro (context, value, expansion, argc, argv);
   else if (m4_is_symbol_value_func (value))
-    {
-      (*m4_get_symbol_value_func (value)) (context, expansion, argc, argv);
-    }
+    m4_get_symbol_value_func (value) (context, expansion, argc, argv);
   else if (m4_is_symbol_value_placeholder (value))
-    {
-      m4_warn (context, 0, M4ARG (0),
-	       _("builtin `%s' requested by frozen file not found"),
-	       m4_get_symbol_value_placeholder (value));
-    }
+    m4_warn (context, 0, M4ARG (0),
+	     _("builtin `%s' requested by frozen file not found"),
+	     m4_get_symbol_value_placeholder (value));
   else
     {
-      assert (!"INTERNAL ERROR: bad symbol type in call_macro ()");
+      assert (!"INTERNAL ERROR: bad symbol type in m4_macro_call ()");
       abort ();
     }
 }
@@ -422,7 +427,7 @@ m4_macro_call (m4 *context, m4_symbol_value *value, m4_obstack *expansion,
    as usual.  */
 static void
 process_macro (m4 *context, m4_symbol_value *value, m4_obstack *obs,
-	       int argc, m4_symbol_value **argv)
+	       int argc, m4_macro_args *argv)
 {
   const char *text;
   int i;
@@ -467,7 +472,7 @@ process_macro (m4 *context, m4_symbol_value *value, m4_obstack *obs,
 
 	case '*':		/* all arguments */
 	case '@':		/* ... same, but quoted */
-	  m4_dump_args (context, obs, argc, argv, ",", *text == '@');
+	  m4_dump_args (context, obs, 1, argv, ",", *text == '@');
 	  text++;
 	  break;
 
@@ -649,7 +654,7 @@ trace_prepre (m4 *context, const char *name, size_t id, m4_symbol_value *value)
    actually expanded.  Used from expand_macro ().  */
 static void
 trace_pre (m4 *context, const char *name, size_t id,
-	   int argc, m4_symbol_value **argv)
+	   int argc, m4_macro_args *argv)
 {
   int i;
 
@@ -670,7 +675,7 @@ trace_pre (m4 *context, const char *name, size_t id,
 	  if (i != 1)
 	    trace_format (context, ", ");
 
-	  m4_symbol_value_print (argv[i], &context->trace_messages,
+	  m4_symbol_value_print (argv->array[i - 1], &context->trace_messages,
 				 quote, lquote, rquote, arg_length, module);
 	}
       trace_format (context, ")");
@@ -681,7 +686,7 @@ trace_pre (m4 *context, const char *name, size_t id,
    expand_macro ().  */
 static void
 trace_post (m4 *context, const char *name, size_t id,
-	    int argc, m4_symbol_value **argv, m4_input_block *expanded,
+	    int argc, m4_macro_args *argv, m4_input_block *expanded,
 	    bool trace_expansion)
 {
   if (trace_expansion)
diff --git a/m4/utility.c b/m4/utility.c
index a8873f3..851da31 100644
--- a/m4/utility.c
+++ b/m4/utility.c
@@ -95,19 +95,23 @@ m4_numeric_arg (m4 *context, const char *caller, const char *arg, int *valuep)
 }
 
 
-/* Print ARGC arguments from the table ARGV to obstack OBS, separated by
-   SEP, and quoted by the current quotes, if QUOTED is true.  */
+/* Print arguments from the table ARGV to obstack OBS, starting at
+   index START, separated by SEP, and quoted by the current quotes, if
+   QUOTED is true.  */
 void
-m4_dump_args (m4 *context, m4_obstack *obs, int argc,
-	      m4_symbol_value **argv, const char *sep, bool quoted)
+m4_dump_args (m4 *context, m4_obstack *obs, int start,
+	      m4_macro_args *argv, const char *sep, bool quoted)
 {
   int i;
   size_t len = strlen (sep);
+  bool need_sep = false;
 
-  for (i = 1; i < argc; i++)
+  for (i = start; i < argv->argc; i++)
     {
-      if (i > 1)
+      if (need_sep)
 	obstack_grow (obs, sep, len);
+      else
+	need_sep = true;
 
       m4_shipout_string (context, obs, M4ARG (i), 0, quoted);
     }
diff --git a/modules/evalparse.c b/modules/evalparse.c
index 260ee1a..31e1e4b 100644
--- a/modules/evalparse.c
+++ b/modules/evalparse.c
@@ -885,7 +885,7 @@ simple_term (m4 *context, eval_token et, number *v1)
 
 /* Main entry point, called from "eval" and "mpeval" builtins.  */
 void
-m4_evaluate (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv)
+m4_evaluate (m4 *context, m4_obstack *obs, int argc, m4_macro_args *argv)
 {
   const char *	me	= M4ARG (0);
   const char *	str	= M4ARG (1);
diff --git a/modules/format.c b/modules/format.c
index 8e5f670..ced3924 100644
--- a/modules/format.c
+++ b/modules/format.c
@@ -30,21 +30,17 @@
    TODO - use xstrtoimax, not atoi, to catch overflow, non-numeric
    arguments, etc.  */
 
-#define ARG_INT(argc, argv) \
-	((argc == 0) ? 0 : \
-	 (--argc, argv++, atoi (M4ARG (-1))))
+#define ARG_INT(i, argc, argv)			\
+  ((argc <= i++) ? 0 : atoi (M4ARG (i - 1)))
 
-#define ARG_LONG(argc, argv) \
-	((argc == 0) ? 0 : \
-	 (--argc, argv++, atol (M4ARG (-1))))
+#define ARG_LONG(i, argc, argv)			\
+  ((argc <= i++) ? 0L : atol (M4ARG (i - 1)))
 
-#define ARG_STR(argc, argv) \
-	((argc == 0) ? "" : \
-	 (--argc, argv++, M4ARG (-1)))
+#define ARG_STR(i, argc, argv)			\
+  ((argc <= i++) ? "" : M4ARG (i - 1))
 
-#define ARG_DOUBLE(argc, argv) \
-	((argc == 0) ? 0 : \
-	 (--argc, argv++, atof (M4ARG (-1))))
+#define ARG_DOUBLE(i, argc, argv)		\
+  ((argc <= i++) ? 0.0 : atof (M4ARG (i - 1)))
 
 
 /* The main formatting function.  Output is placed on the obstack OBS,
@@ -54,14 +50,15 @@
    format.  */
 
 static void
-format (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv)
+format (m4 *context, m4_obstack *obs, int argc, m4_macro_args *argv)
 {
-  const char *name = ARG_STR (argc, argv);	/* macro name */
-  const char *f;			/* format control string */
-  const char *fmt;			/* position within f */
-  char fstart[] = "%'+- 0#*.*hhd";	/* current format spec */
-  char *p;				/* position within fstart */
-  unsigned char c;			/* a simple character */
+  const char *name = M4ARG (0);		/* Macro name.  */
+  const char *f;			/* Format control string.  */
+  const char *fmt;			/* Position within f.  */
+  char fstart[] = "%'+- 0#*.*hhd";	/* Current format spec.  */
+  char *p;				/* Position within fstart.  */
+  unsigned char c;			/* A simple character.  */
+  int index = 1;			/* Index within argc used so far.  */
 
   /* Flags.  */
   char flags;				/* flags to use in fstart */
@@ -89,7 +86,7 @@ format (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv)
   char *str;			/* malloc'd buffer of formatted text */
   enum {CHAR, INT, LONG, DOUBLE, STR} datatype;
 
-  f = fmt = ARG_STR (argc, argv);
+  f = fmt = ARG_STR (index, argc, argv);
   memset (ok, 0, sizeof ok);
   for (;;)
     {
@@ -174,7 +171,7 @@ format (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv)
       *p++ = '*';
       if (*fmt == '*')
 	{
-	  width = ARG_INT (argc, argv);
+	  width = ARG_INT (index, argc, argv);
 	  fmt++;
 	}
       else
@@ -194,7 +191,7 @@ format (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv)
 	  ok['c'] = 0;
 	  if (*(++fmt) == '*')
 	    {
-	      prec = ARG_INT (argc, argv);
+	      prec = ARG_INT (index, argc, argv);
 	      ++fmt;
 	    }
 	  else
@@ -279,23 +276,24 @@ format (m4 *context, m4_obstack *obs, int argc, m4_symbol_value **argv)
       switch (datatype)
 	{
 	case CHAR:
-	  str = xasprintf (fstart, width, ARG_INT (argc, argv));
+	  str = xasprintf (fstart, width, ARG_INT (index, argc, argv));
 	  break;
 
 	case INT:
-	  str = xasprintf (fstart, width, prec, ARG_INT (argc, argv));
+	  str = xasprintf (fstart, width, prec, ARG_INT (index, argc, argv));
 	  break;
 
 	case LONG:
-	  str = xasprintf (fstart, width, prec, ARG_LONG (argc, argv));
+	  str = xasprintf (fstart, width, prec, ARG_LONG (index, argc, argv));
 	  break;
 
 	case DOUBLE:
-	  str = xasprintf (fstart, width, prec, ARG_DOUBLE (argc, argv));
+	  str = xasprintf (fstart, width, prec,
+			   ARG_DOUBLE (index, argc, argv));
 	  break;
 
 	case STR:
-	  str = xasprintf (fstart, width, prec, ARG_STR (argc, argv));
+	  str = xasprintf (fstart, width, prec, ARG_STR (index, argc, argv));
 	  break;
 
 	default:
diff --git a/modules/gnu.c b/modules/gnu.c
index e857632..6fd6623 100644
--- a/modules/gnu.c
+++ b/modules/gnu.c
@@ -409,14 +409,14 @@ M4BUILTIN_HANDLER (builtin)
   const char *name;
   m4_symbol_value *value;
 
-  if (!m4_is_symbol_value_text (argv[1]))
+  if (!m4_is_symbol_value_text (argv->array[1 - 1]))
     {
-      if (m4_is_symbol_value_func (argv[1])
-	  && m4_get_symbol_value_func (argv[1]) == builtin_builtin)
+      if (m4_is_symbol_value_func (argv->array[1 - 1])
+	  && m4_get_symbol_value_func (argv->array[1 - 1]) == builtin_builtin)
 	{
 	  if (m4_bad_argc (context, argc, me, 2, 2, false))
 	    return;
-	  if (!m4_is_symbol_value_text (argv[2]))
+	  if (!m4_is_symbol_value_text (argv->array[2 - 1]))
 	    {
 	      m4_warn (context, 0, me, _("invalid macro name ignored"));
 	      return;
@@ -448,11 +448,22 @@ M4BUILTIN_HANDLER (builtin)
 			    (bp->flags & M4_BUILTIN_SIDE_EFFECT) != 0))
 	    {
 	      int i;
+	      /* TODO - make use of $@ reference */
+	      m4_macro_args *new_argv;
+	      new_argv = xmalloc (offsetof (m4_macro_args, array)
+				  + ((argc - 2) * sizeof (m4_symbol_value *)));
+	      new_argv->argc = argc - 1;
+	      new_argv->inuse = false;
+	      new_argv->argv0 = name;
+	      new_argv->arraylen = argc - 2;
+	      memcpy (&new_argv->array[0], &argv->array[1],
+		      (argc - 2) * sizeof (m4_symbol_value *));
 	      if ((bp->flags & M4_BUILTIN_GROKS_MACRO) == 0)
 		for (i = 2; i < argc; i++)
-		  if (!m4_is_symbol_value_text (argv[i]))
-		    m4_set_symbol_value_text (argv[i], "");
-	      bp->func (context, obs, argc - 1, argv + 1);
+		  if (!m4_is_symbol_value_text (argv->array[i - 1]))
+		    m4_set_symbol_value_text (new_argv->array[i - 2], "");
+	      bp->func (context, obs, argc - 1, new_argv);
+	      free (new_argv);
 	    }
 	  free (value);
 	}
@@ -667,24 +678,36 @@ M4BUILTIN_HANDLER (format)
  **/
 M4BUILTIN_HANDLER (indir)
 {
-  if (!m4_is_symbol_value_text (argv[1]))
-    m4_warn (context, 0, M4ARG (0), _("invalid macro name ignored"));
+  const char *me = M4ARG (0);
+  if (!m4_is_symbol_value_text (argv->array[1 - 1]))
+    m4_warn (context, 0, me, _("invalid macro name ignored"));
   else
     {
       const char *name = M4ARG (1);
       m4_symbol *symbol = m4_symbol_lookup (M4SYMTAB, name);
 
       if (symbol == NULL)
-	m4_warn (context, 0, M4ARG (0), _("undefined macro `%s'"), name);
+	m4_warn (context, 0, me, _("undefined macro `%s'"), name);
       else
 	{
 	  int i;
+	  /* TODO - make use of $@ reference */
+	  m4_macro_args *new_argv;
+	  new_argv = xmalloc (offsetof (m4_macro_args, array)
+			      + ((argc - 2) * sizeof (m4_symbol_value *)));
+	  new_argv->argc = argc - 1;
+	  new_argv->inuse = false;
+	  new_argv->argv0 = name;
+	  new_argv->arraylen = argc - 2;
+	  memcpy (&new_argv->array[0], &argv->array[1],
+		  (argc - 2) * sizeof (m4_symbol_value *));
 	  if (!m4_symbol_groks_macro (symbol))
 	    for (i = 2; i < argc; i++)
-	      if (!m4_is_symbol_value_text (argv[i]))
-		m4_set_symbol_value_text (argv[i], "");
+	      if (!m4_is_symbol_value_text (argv->array[i - 1]))
+		m4_set_symbol_value_text (new_argv->array[i - 2], "");
 	  m4_macro_call (context, m4_get_symbol_value (symbol), obs,
-			 argc - 1, argv + 1);
+			 argc - 1, new_argv);
+	  free (new_argv);
 	}
     }
 }
diff --git a/modules/m4.c b/modules/m4.c
index bffb274..7c0d8ed 100644
--- a/modules/m4.c
+++ b/modules/m4.c
@@ -50,7 +50,7 @@
 extern void m4_set_sysval    (int value);
 extern void m4_sysval_flush  (m4 *context, bool report);
 extern void m4_dump_symbols  (m4 *context, m4_dump_symbol_data *data, int argc,
-			      m4_symbol_value **argv, bool complain);
+			      m4_macro_args *argv, bool complain);
 extern const char *m4_expand_ranges (const char *s, m4_obstack *obs);
 extern void m4_make_temp     (m4 *context, m4_obstack *obs, const char *macro,
 			      const char *name, bool dir);
@@ -102,7 +102,7 @@ extern void m4_make_temp     (m4 *context, m4_obstack *obs, const char *macro,
 typedef intmax_t number;
 typedef uintmax_t unumber;
 
-static void	include		(m4 *context, int argc, m4_symbol_value **argv,
+static void	include		(m4 *context, int argc, m4_macro_args *argv,
 				 bool silent);
 static int	dumpdef_cmp_CB	(const void *s1, const void *s2);
 static void *	dump_symbol_CB  (m4_symbol_table *ignored, const char *name,
@@ -113,7 +113,7 @@ static void	numb_obstack	(m4_obstack *obs, number value,
 
 
 /* Generate prototypes for each builtin handler function. */
-#define BUILTIN(handler, macros,  blind, side, min, max) M4BUILTIN(handler)
+#define BUILTIN(handler, macros,  blind, side, min, max) M4BUILTIN (handler)
   builtin_functions
 #undef BUILTIN
 
@@ -122,7 +122,7 @@ static void	numb_obstack	(m4_obstack *obs, number value,
 m4_builtin m4_builtin_table[] =
 {
 #define BUILTIN(handler, macros, blind, side, min, max)	\
-  { CONC(builtin_, handler), STR(handler),		\
+  { CONC (builtin_, handler), STR (handler),		\
     ((macros ? M4_BUILTIN_GROKS_MACRO : 0)		\
      | (blind ? M4_BUILTIN_BLIND : 0)			\
      | (side ? M4_BUILTIN_SIDE_EFFECT : 0)),		\
@@ -160,14 +160,14 @@ M4INIT_HANDLER (m4)
 
 M4BUILTIN_HANDLER (define)
 {
-  if (m4_is_symbol_value_text (argv[1]))
+  if (m4_is_symbol_value_text (argv->array[1 - 1]))
     {
       m4_symbol_value *value = m4_symbol_value_create ();
 
       if (argc == 2)
 	m4_set_symbol_value_text (value, xstrdup (""));
       else
-	m4_symbol_value_copy (value, argv[2]);
+	m4_symbol_value_copy (value, argv->array[2 - 1]);
 
       m4_symbol_define (M4SYMTAB, M4ARG (1), value);
     }
@@ -191,14 +191,14 @@ M4BUILTIN_HANDLER (undefine)
 
 M4BUILTIN_HANDLER (pushdef)
 {
-  if (m4_is_symbol_value_text (argv[1]))
+  if (m4_is_symbol_value_text (argv->array[1 - 1]))
     {
       m4_symbol_value *value = m4_symbol_value_create ();
 
       if (argc == 2)
 	m4_set_symbol_value_text (value, xstrdup (""));
       else
-	m4_symbol_value_copy (value, argv[2]);
+	m4_symbol_value_copy (value, argv->array[2 - 1]);
 
       m4_symbol_pushdef (M4SYMTAB, M4ARG (1), value);
     }
@@ -248,6 +248,7 @@ M4BUILTIN_HANDLER (ifelse)
 {
   const char *me = M4ARG (0);
   const char *result;
+  int index;
 
   /* The valid ranges of argc for ifelse is discontinuous, we cannot
      rely on the regular mechanisms.  */
@@ -260,14 +261,14 @@ M4BUILTIN_HANDLER (ifelse)
     /* Diagnose excess arguments if 5, 8, 11, etc., actual arguments.  */
     m4_bad_argc (context, argc, me, 0, argc - 2, false);
 
-  argv++;
+  index = 1;
   argc--;
 
   result = NULL;
   while (result == NULL)
 
-    if (strcmp (M4ARG (0), M4ARG (1)) == 0)
-      result = M4ARG (2);
+    if (strcmp (M4ARG (index), M4ARG (index + 1)) == 0)
+      result = M4ARG (index + 2);
 
     else
       switch (argc)
@@ -277,12 +278,12 @@ M4BUILTIN_HANDLER (ifelse)
 
 	case 4:
 	case 5:
-	  result = M4ARG (3);
+	  result = M4ARG (index + 3);
 	  break;
 
 	default:
 	  argc -= 3;
-	  argv += 3;
+	  index += 3;
 	}
 
   obstack_grow (obs, result, strlen (result));
@@ -327,7 +328,7 @@ dump_symbol_CB (m4_symbol_table *ignored, const char *name, m4_symbol *symbol,
    symbols, otherwise, only the specified symbols.  */
 void
 m4_dump_symbols (m4 *context, m4_dump_symbol_data *data, int argc,
-		 m4_symbol_value **argv, bool complain)
+		 m4_macro_args *argv, bool complain)
 {
   assert (obstack_object_size (data->obs) == 0);
   data->size = obstack_room (data->obs) / sizeof (const char *);
@@ -394,15 +395,16 @@ M4BUILTIN_HANDLER (dumpdef)
    macro-definition token on the input stack.  */
 M4BUILTIN_HANDLER (defn)
 {
+  const char *me = M4ARG (0);
   int i;
 
   for (i = 1; i < argc; i++)
     {
-      const char *name = m4_get_symbol_value_text (argv[i]);
+      const char *name = M4ARG (i);
       m4_symbol *symbol = m4_symbol_lookup (M4SYMTAB, name);
 
       if (!symbol)
-	m4_warn (context, 0, M4ARG (0), _("undefined macro `%s'"), name);
+	m4_warn (context, 0, me, _("undefined macro `%s'"), name);
       else if (m4_is_symbol_text (symbol))
 	m4_shipout_string (context, obs, m4_get_symbol_text (symbol), 0, true);
       else if (m4_is_symbol_func (symbol))
@@ -647,7 +649,7 @@ M4BUILTIN_HANDLER (dnl)
    output argument is quoted with the current quotes.  */
 M4BUILTIN_HANDLER (shift)
 {
-  m4_dump_args (context, obs, argc - 1, argv + 1, ",", true);
+  m4_dump_args (context, obs, 2, argv, ",", true);
 }
 
 /* Change the current quotes.  The function set_quotes () lives in
@@ -676,7 +678,7 @@ M4BUILTIN_HANDLER (changecom)
 /* Generic include function.  Include the file given by the first argument,
    if it exists.  Complain about inaccesible files iff SILENT is false.  */
 static void
-include (m4 *context, int argc, m4_symbol_value **argv, bool silent)
+include (m4 *context, int argc, m4_macro_args *argv, bool silent)
 {
   FILE *fp;
   char *name = NULL;
@@ -806,7 +808,7 @@ M4BUILTIN_HANDLER (mkstemp)
 M4BUILTIN_HANDLER (errprint)
 {
   assert (obstack_object_size (obs) == 0);
-  m4_dump_args (context, obs, argc, argv, " ", false);
+  m4_dump_args (context, obs, 1, argv, " ", false);
   obstack_1grow (obs, '\0');
   m4_sysval_flush (context, false);
   fputs ((char *) obstack_finish (obs), stderr);
@@ -861,7 +863,7 @@ M4BUILTIN_HANDLER (m4wrap)
   if (m4_get_posixly_correct_opt (context))
     m4_shipout_string (context, obs, M4ARG (1), 0, false);
   else
-    m4_dump_args (context, obs, argc, argv, " ", false);
+    m4_dump_args (context, obs, 1, argv, " ", false);
   obstack_1grow (obs, '\0');
   m4_push_wrapup (context, obstack_finish (obs));
 }
diff --git a/modules/m4.h b/modules/m4.h
index 5c14e25..4214aed 100644
--- a/modules/m4.h
+++ b/modules/m4.h
@@ -39,7 +39,7 @@ typedef struct
 typedef void m4_sysval_flush_func (m4 *context, bool report);
 typedef void m4_set_sysval_func (int value);
 typedef void m4_dump_symbols_func (m4 *context, m4_dump_symbol_data *data,
-				   int argc, m4_symbol_value **argv,
+				   int argc, m4_macro_args *argv,
 				   bool complain);
 typedef const char *m4_expand_ranges_func (const char *s, m4_obstack *obs);
 typedef void m4_make_temp_func (m4 *context, m4_obstack *obs,
-- 
1.5.3.5

From ac8845562f0066ef07d74a1f5b0e749eceb19b89 Mon Sep 17 00:00:00 2001
From: Eric Blake <[email protected]>
Date: Fri, 19 Oct 2007 07:43:34 -0600
Subject: [PATCH] Stage 1: convert token_data** into new object.

* m4/gnulib-cache.m4: Import flexmember module.
* src/m4.h (struct macro_arguments, struct token_chain): New
structs.
(builtin_func): Alter signature.
(token_data): Add new TOKEN_COMP alternative.
* src/builtin.c: All builtins changed.
(ARG, dump_args, define_macro, expand_user_macro): Update to use
struct.
* src/debug.c (trace_pre, trace_post): Likewise.
* src/format.c (ARG_INT, ARG_LONG, ARG_STR, ARG_DOUBLE, format):
Likewise.
* src/macro.c (collect_arguments): Build new struct.
(call_macro, expand_macro): Update to use new struct.

(cherry picked from commit 44f5da7de32ac8f71f26d9e441316fa563db30d6)

Signed-off-by: Eric Blake <[email protected]>
---
 ChangeLog          |   17 +++++
 m4/gnulib-cache.m4 |    4 +-
 src/builtin.c      |  201 ++++++++++++++++++++++++++++++----------------------
 src/debug.c        |   10 ++--
 src/format.c       |   45 +++++-------
 src/m4.h           |   71 ++++++++++++++-----
 src/macro.c        |   51 ++++++++------
 7 files changed, 243 insertions(+), 156 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9bbf726..14deb9f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+2007-11-24  Eric Blake  <[email protected]>
+
+	Stage 1: convert token_data** into new object.
+	* m4/gnulib-cache.m4: Import flexmember module.
+	* src/m4.h (struct macro_arguments, struct token_chain): New
+	structs.
+	(builtin_func): Alter signature.
+	(token_data): Add new TOKEN_COMP alternative.
+	* src/builtin.c: All builtins changed.
+	(ARG, dump_args, define_macro, expand_user_macro): Update to use
+	struct.
+	* src/debug.c (trace_pre, trace_post): Likewise.
+	* src/format.c (ARG_INT, ARG_LONG, ARG_STR, ARG_DOUBLE, format):
+	Likewise.
+	* src/macro.c (collect_arguments): Build new struct.
+	(call_macro, expand_macro): Update to use new struct.
+
 2007-11-22  Eric Blake  <[email protected]>
 
 	More error messages tied to macro names.
diff --git a/m4/gnulib-cache.m4 b/m4/gnulib-cache.m4
index 4d1727d..a89650c 100644
--- a/m4/gnulib-cache.m4
+++ b/m4/gnulib-cache.m4
@@ -15,11 +15,11 @@
 
 
 # Specification in the form of a command-line invocation:
-#   gnulib-tool --import --dir=. --local-dir=local --lib=libm4 --source-base=lib --m4-base=m4 --doc-base=doc --aux-dir=build-aux --with-tests --no-libtool --macro-prefix=M4 assert avltree-oset binary-io clean-temp cloexec close-stream closein config-h error fdl fflush fopen-safer free fseeko gendocs getopt gnupload gpl-3.0 mkstemp obstack regex stdbool stdint stdlib-safer strtol unlocked-io verror version-etc version-etc-fsf xalloc xprintf xvasprintf-posix
+#   gnulib-tool --import --dir=. --local-dir=local --lib=libm4 --source-base=lib --m4-base=m4 --doc-base=doc --aux-dir=build-aux --with-tests --no-libtool --macro-prefix=M4 assert avltree-oset binary-io clean-temp cloexec close-stream closein config-h error fdl fflush flexmember fopen-safer free fseeko gendocs getopt gnupload gpl-3.0 mkstemp obstack regex stdbool stdint stdlib-safer strtol unlocked-io verror version-etc version-etc-fsf xalloc xprintf xvasprintf-posix
 
 # Specification in the form of a few gnulib-tool.m4 macro invocations:
 gl_LOCAL_DIR([local])
-gl_MODULES([assert avltree-oset binary-io clean-temp cloexec close-stream closein config-h error fdl fflush fopen-safer free fseeko gendocs getopt gnupload gpl-3.0 mkstemp obstack regex stdbool stdint stdlib-safer strtol unlocked-io verror version-etc version-etc-fsf xalloc xprintf xvasprintf-posix])
+gl_MODULES([assert avltree-oset binary-io clean-temp cloexec close-stream closein config-h error fdl fflush flexmember fopen-safer free fseeko gendocs getopt gnupload gpl-3.0 mkstemp obstack regex stdbool stdint stdlib-safer strtol unlocked-io verror version-etc version-etc-fsf xalloc xprintf xvasprintf-posix])
 gl_AVOID([])
 gl_SOURCE_BASE([lib])
 gl_M4_BASE([m4])
diff --git a/src/builtin.c b/src/builtin.c
index cc4e469..eb66465 100644
--- a/src/builtin.c
+++ b/src/builtin.c
@@ -30,14 +30,16 @@
 # include <sys/wait.h>
 #endif
 
-#define ARG(i)	(argc > (i) ? TOKEN_DATA_TEXT (argv[i]) : "")
+#define ARG(i)								\
+  ((i) == 0 ? argv->argv0						\
+   : argv->argc > (i) ? TOKEN_DATA_TEXT (argv->array[(i) - 1]) : "")
 
 /* Initialization of builtin and predefined macros.  The table
    "builtin_tab" is both used for initialization, and by the "builtin"
    builtin.  */
 
 #define DECLARE(name) \
-  static void name (struct obstack *, int, token_data **)
+  static void name (struct obstack *, int, macro_arguments *)
 
 DECLARE (m4___file__);
 DECLARE (m4___line__);
@@ -602,16 +604,19 @@ shipout_int (struct obstack *obs, int val)
 `----------------------------------------------------------------------*/
 
 static void
-dump_args (struct obstack *obs, int argc, token_data **argv,
+dump_args (struct obstack *obs, int start, macro_arguments *argv,
 	   const char *sep, bool quoted)
 {
   int i;
+  bool dump_sep = false;
   size_t len = strlen (sep);
 
-  for (i = 1; i < argc; i++)
+  for (i = start; i < argv->argc; i++)
     {
-      if (i > 1)
+      if (dump_sep)
 	obstack_grow (obs, sep, len);
+      else
+	dump_sep = true;
       if (quoted)
 	obstack_grow (obs, lquote.string, lquote.length);
       obstack_grow (obs, ARG (i), strlen (ARG (i)));
@@ -623,14 +628,15 @@ dump_args (struct obstack *obs, int argc, token_data **argv,
 /* The rest of this file is code for builtins and expansion of user
    defined macros.  All the functions for builtins have a prototype as:
 
-	void m4_MACRONAME (struct obstack *obs, int argc, char *argv[]);
+     void m4_MACRONAME (struct obstack *obs, int argc, macro_arguments *argv);
 
-   The function are expected to leave their expansion on the obstack OBS,
-   as an unfinished object.  ARGV is a table of ARGC pointers to the
-   individual arguments to the macro.  Please note that in general
-   argv[argc] != NULL.  */
+   The functions are expected to leave their expansion on the obstack OBS,
+   as an unfinished object.  ARGV is an object representing ARGC pointers
+   to the individual arguments to the macro; the object may be compressed
+   due to references to $@ expansions, so accessors should be used.  Please
+   note that in general argv[argc] != NULL.  */
 
-/* The first section are macros for definining, undefining, examining,
+/* The first section are macros for defining, undefining, examining,
    changing, ... other macros.  */
 
 /*-------------------------------------------------------------------------.
@@ -641,7 +647,7 @@ dump_args (struct obstack *obs, int argc, token_data **argv,
 `-------------------------------------------------------------------------*/
 
 static void
-define_macro (int argc, token_data **argv, symbol_lookup mode)
+define_macro (int argc, macro_arguments *argv, symbol_lookup mode)
 {
   const builtin *bp;
   const char *me = ARG (0);
@@ -649,7 +655,7 @@ define_macro (int argc, token_data **argv, symbol_lookup mode)
   if (bad_argc (me, argc, 1, 2))
     return;
 
-  if (TOKEN_DATA_TYPE (argv[1]) != TOKEN_TEXT)
+  if (TOKEN_DATA_TYPE (argv->array[0]) != TOKEN_TEXT)
     {
       m4_warn (0, me, _("invalid macro name ignored"));
       return;
@@ -661,14 +667,14 @@ define_macro (int argc, token_data **argv, symbol_lookup mode)
       return;
     }
 
-  switch (TOKEN_DATA_TYPE (argv[2]))
+  switch (TOKEN_DATA_TYPE (argv->array[1]))
     {
     case TOKEN_TEXT:
       define_user_macro (ARG (1), ARG (2), mode);
       break;
 
     case TOKEN_FUNC:
-      bp = find_builtin_by_addr (TOKEN_DATA_FUNC (argv[2]));
+      bp = find_builtin_by_addr (TOKEN_DATA_FUNC (argv->array[1]));
       if (bp == NULL)
 	return;
       else
@@ -682,13 +688,13 @@ define_macro (int argc, token_data **argv, symbol_lookup mode)
 }
 
 static void
-m4_define (struct obstack *obs, int argc, token_data **argv)
+m4_define (struct obstack *obs, int argc, macro_arguments *argv)
 {
   define_macro (argc, argv, SYMBOL_INSERT);
 }
 
 static void
-m4_undefine (struct obstack *obs, int argc, token_data **argv)
+m4_undefine (struct obstack *obs, int argc, macro_arguments *argv)
 {
   int i;
   if (bad_argc (ARG (0), argc, 1, -1))
@@ -698,13 +704,13 @@ m4_undefine (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4_pushdef (struct obstack *obs, int argc, token_data **argv)
+m4_pushdef (struct obstack *obs, int argc, macro_arguments *argv)
 {
-  define_macro (argc, argv,  SYMBOL_PUSHDEF);
+  define_macro (argc, argv, SYMBOL_PUSHDEF);
 }
 
 static void
-m4_popdef (struct obstack *obs, int argc, token_data **argv)
+m4_popdef (struct obstack *obs, int argc, macro_arguments *argv)
 {
   int i;
   if (bad_argc (ARG (0), argc, 1, -1))
@@ -718,7 +724,7 @@ m4_popdef (struct obstack *obs, int argc, token_data **argv)
 `---------------------*/
 
 static void
-m4_ifdef (struct obstack *obs, int argc, token_data **argv)
+m4_ifdef (struct obstack *obs, int argc, macro_arguments *argv)
 {
   symbol *s;
   const char *result;
@@ -739,10 +745,11 @@ m4_ifdef (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4_ifelse (struct obstack *obs, int argc, token_data **argv)
+m4_ifelse (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *result;
   const char *me;
+  int index;
 
   if (argc == 2)
     return;
@@ -754,14 +761,14 @@ m4_ifelse (struct obstack *obs, int argc, token_data **argv)
     /* Diagnose excess arguments if 5, 8, 11, etc., actual arguments.  */
     bad_argc (me, argc, 0, argc - 2);
 
-  argv++;
+  index = 1;
   argc--;
 
   result = NULL;
   while (result == NULL)
 
-    if (strcmp (ARG (0), ARG (1)) == 0)
-      result = ARG (2);
+    if (strcmp (ARG (index), ARG (index + 1)) == 0)
+      result = ARG (index + 2);
 
     else
       switch (argc)
@@ -771,12 +778,12 @@ m4_ifelse (struct obstack *obs, int argc, token_data **argv)
 
 	case 4:
 	case 5:
-	  result = ARG (3);
+	  result = ARG (index + 3);
 	  break;
 
 	default:
 	  argc -= 3;
-	  argv += 3;
+	  index += 3;
 	}
 
   obstack_grow (obs, result, strlen (result));
@@ -826,7 +833,7 @@ dumpdef_cmp (const void *s1, const void *s2)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_dumpdef (struct obstack *obs, int argc, token_data **argv)
+m4_dumpdef (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   symbol *s;
@@ -900,7 +907,7 @@ m4_dumpdef (struct obstack *obs, int argc, token_data **argv)
 `---------------------------------------------------------------------*/
 
 static void
-m4_builtin (struct obstack *obs, int argc, token_data **argv)
+m4_builtin (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   const builtin *bp;
@@ -908,7 +915,7 @@ m4_builtin (struct obstack *obs, int argc, token_data **argv)
 
   if (bad_argc (me, argc, 1, -1))
     return;
-  if (TOKEN_DATA_TYPE (argv[1]) != TOKEN_TEXT)
+  if (TOKEN_DATA_TYPE (argv->array[0]) != TOKEN_TEXT)
     {
       m4_warn (0, me, _("invalid macro name ignored"));
       return;
@@ -921,14 +928,25 @@ m4_builtin (struct obstack *obs, int argc, token_data **argv)
   else
     {
       int i;
+      /* TODO make use of $@ reference, instead of copying argv.  */
+      macro_arguments *new_argv = xmalloc (offsetof (macro_arguments, array)
+					   + ((argc - 2)
+					      * sizeof (token_data *)));
+      new_argv->argc = argc - 1;
+      new_argv->inuse = false;
+      new_argv->argv0 = name;
+      new_argv->arraylen = argc - 2;
+      memcpy (&new_argv->array[0], &argv->array[1],
+	      (argc - 2) * sizeof (token_data *));
       if (!bp->groks_macro_args)
 	for (i = 2; i < argc; i++)
-	  if (TOKEN_DATA_TYPE (argv[i]) != TOKEN_TEXT)
+	  if (TOKEN_DATA_TYPE (new_argv->array[i - 2]) != TOKEN_TEXT)
 	    {
-	      TOKEN_DATA_TYPE (argv[i]) = TOKEN_TEXT;
-	      TOKEN_DATA_TEXT (argv[i]) = (char *) "";
+	      TOKEN_DATA_TYPE (new_argv->array[i - 2]) = TOKEN_TEXT;
+	      TOKEN_DATA_TEXT (new_argv->array[i - 2]) = (char *) "";
 	    }
-      bp->func (obs, argc - 1, argv + 1);
+      bp->func (obs, argc - 1, new_argv);
+      free (new_argv);
     }
 }
 
@@ -940,7 +958,7 @@ m4_builtin (struct obstack *obs, int argc, token_data **argv)
 `------------------------------------------------------------------------*/
 
 static void
-m4_indir (struct obstack *obs, int argc, token_data **argv)
+m4_indir (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   symbol *s;
@@ -948,7 +966,7 @@ m4_indir (struct obstack *obs, int argc, token_data **argv)
 
   if (bad_argc (me, argc, 1, -1))
     return;
-  if (TOKEN_DATA_TYPE (argv[1]) != TOKEN_TEXT)
+  if (TOKEN_DATA_TYPE (argv->array[0]) != TOKEN_TEXT)
     {
       m4_warn (0, me, _("invalid macro name ignored"));
       return;
@@ -961,14 +979,25 @@ m4_indir (struct obstack *obs, int argc, token_data **argv)
   else
     {
       int i;
+      /* TODO make use of $@ reference, instead of copying argv.  */
+      macro_arguments *new_argv = xmalloc (offsetof (macro_arguments, array)
+					   + ((argc - 2)
+					      * sizeof (token_data *)));
+      new_argv->argc = argc - 1;
+      new_argv->inuse = false;
+      new_argv->argv0 = name;
+      new_argv->arraylen = argc - 2;
+      memcpy (&new_argv->array[0], &argv->array[1],
+	      (argc - 2) * sizeof (token_data *));
       if (!SYMBOL_MACRO_ARGS (s))
 	for (i = 2; i < argc; i++)
-	  if (TOKEN_DATA_TYPE (argv[i]) != TOKEN_TEXT)
+	  if (TOKEN_DATA_TYPE (new_argv->array[i - 2]) != TOKEN_TEXT)
 	    {
-	      TOKEN_DATA_TYPE (argv[i]) = TOKEN_TEXT;
-	      TOKEN_DATA_TEXT (argv[i]) = (char *) "";
+	      TOKEN_DATA_TYPE (new_argv->array[i - 2]) = TOKEN_TEXT;
+	      TOKEN_DATA_TEXT (new_argv->array[i - 2]) = (char *) "";
 	    }
-      call_macro (s, argc - 1, argv + 1, obs);
+      call_macro (s, argc - 1, new_argv, obs);
+      free (new_argv);
     }
 }
 
@@ -979,7 +1008,7 @@ m4_indir (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_defn (struct obstack *obs, int argc, token_data **argv)
+m4_defn (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   symbol *s;
@@ -1060,7 +1089,7 @@ m4_defn (struct obstack *obs, int argc, token_data **argv)
 static int sysval;
 
 static void
-m4_syscmd (struct obstack *obs, int argc, token_data **argv)
+m4_syscmd (struct obstack *obs, int argc, macro_arguments *argv)
 {
   if (bad_argc (ARG (0), argc, 1, 1))
     {
@@ -1084,7 +1113,7 @@ m4_syscmd (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4_esyscmd (struct obstack *obs, int argc, token_data **argv)
+m4_esyscmd (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   FILE *pin;
@@ -1114,7 +1143,7 @@ m4_esyscmd (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4_sysval (struct obstack *obs, int argc, token_data **argv)
+m4_sysval (struct obstack *obs, int argc, macro_arguments *argv)
 {
   shipout_int (obs, (sysval == -1 ? 127
 		     : (M4SYSVAL_EXITBITS (sysval)
@@ -1127,7 +1156,7 @@ m4_sysval (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_eval (struct obstack *obs, int argc, token_data **argv)
+m4_eval (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   int32_t value = 0;
@@ -1190,7 +1219,7 @@ m4_eval (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4_incr (struct obstack *obs, int argc, token_data **argv)
+m4_incr (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   int value;
@@ -1205,7 +1234,7 @@ m4_incr (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4_decr (struct obstack *obs, int argc, token_data **argv)
+m4_decr (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   int value;
@@ -1228,7 +1257,7 @@ m4_decr (struct obstack *obs, int argc, token_data **argv)
 `-----------------------------------------------------------------------*/
 
 static void
-m4_divert (struct obstack *obs, int argc, token_data **argv)
+m4_divert (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   int i = 0;
@@ -1245,7 +1274,7 @@ m4_divert (struct obstack *obs, int argc, token_data **argv)
 `-----------------------------------------------------*/
 
 static void
-m4_divnum (struct obstack *obs, int argc, token_data **argv)
+m4_divnum (struct obstack *obs, int argc, macro_arguments *argv)
 {
   bad_argc (ARG (0), argc, 0, 0);
   shipout_int (obs, current_diversion);
@@ -1259,7 +1288,7 @@ m4_divnum (struct obstack *obs, int argc, token_data **argv)
 `-----------------------------------------------------------------------*/
 
 static void
-m4_undivert (struct obstack *obs, int argc, token_data **argv)
+m4_undivert (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   int i;
@@ -1303,7 +1332,7 @@ m4_undivert (struct obstack *obs, int argc, token_data **argv)
 `------------------------------------------------------------------------*/
 
 static void
-m4_dnl (struct obstack *obs, int argc, token_data **argv)
+m4_dnl (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
 
@@ -1317,11 +1346,12 @@ m4_dnl (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_shift (struct obstack *obs, int argc, token_data **argv)
+m4_shift (struct obstack *obs, int argc, macro_arguments *argv)
 {
   if (bad_argc (ARG (0), argc, 1, -1))
     return;
-  dump_args (obs, argc - 1, argv + 1, ",", true);
+  /* TODO push a $@ reference.  */
+  dump_args (obs, 2, argv, ",", true);
 }
 
 /*--------------------------------------------------------------------------.
@@ -1329,13 +1359,13 @@ m4_shift (struct obstack *obs, int argc, token_data **argv)
 `--------------------------------------------------------------------------*/
 
 static void
-m4_changequote (struct obstack *obs, int argc, token_data **argv)
+m4_changequote (struct obstack *obs, int argc, macro_arguments *argv)
 {
   bad_argc (ARG (0), argc, 0, 2);
 
   /* Explicit NULL distinguishes between empty and missing argument.  */
   set_quotes ((argc >= 2) ? ARG (1) : NULL,
-	     (argc >= 3) ? ARG (2) : NULL);
+	      (argc >= 3) ? ARG (2) : NULL);
 }
 
 /*--------------------------------------------------------------------.
@@ -1344,7 +1374,7 @@ m4_changequote (struct obstack *obs, int argc, token_data **argv)
 `--------------------------------------------------------------------*/
 
 static void
-m4_changecom (struct obstack *obs, int argc, token_data **argv)
+m4_changecom (struct obstack *obs, int argc, macro_arguments *argv)
 {
   bad_argc (ARG (0), argc, 0, 2);
 
@@ -1361,7 +1391,7 @@ m4_changecom (struct obstack *obs, int argc, token_data **argv)
 `-----------------------------------------------------------------------*/
 
 static void
-m4_changeword (struct obstack *obs, int argc, token_data **argv)
+m4_changeword (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
 
@@ -1382,7 +1412,7 @@ m4_changeword (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-include (int argc, token_data **argv, bool silent)
+include (int argc, macro_arguments *argv, bool silent)
 {
   const char *me = ARG (0);
   FILE *fp;
@@ -1408,7 +1438,7 @@ include (int argc, token_data **argv, bool silent)
 `------------------------------------------------*/
 
 static void
-m4_include (struct obstack *obs, int argc, token_data **argv)
+m4_include (struct obstack *obs, int argc, macro_arguments *argv)
 {
   include (argc, argv, false);
 }
@@ -1418,7 +1448,7 @@ m4_include (struct obstack *obs, int argc, token_data **argv)
 `----------------------------------*/
 
 static void
-m4_sinclude (struct obstack *obs, int argc, token_data **argv)
+m4_sinclude (struct obstack *obs, int argc, macro_arguments *argv)
 {
   include (argc, argv, true);
 }
@@ -1462,7 +1492,7 @@ mkstemp_helper (struct obstack *obs, const char *me, const char *name)
 }
 
 static void
-m4_maketemp (struct obstack *obs, int argc, token_data **argv)
+m4_maketemp (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
 
@@ -1507,7 +1537,7 @@ m4_maketemp (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4_mkstemp (struct obstack *obs, int argc, token_data **argv)
+m4_mkstemp (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
 
@@ -1521,11 +1551,11 @@ m4_mkstemp (struct obstack *obs, int argc, token_data **argv)
 `----------------------------------------*/
 
 static void
-m4_errprint (struct obstack *obs, int argc, token_data **argv)
+m4_errprint (struct obstack *obs, int argc, macro_arguments *argv)
 {
   if (bad_argc (ARG (0), argc, 1, -1))
     return;
-  dump_args (obs, argc, argv, " ", false);
+  dump_args (obs, 1, argv, " ", false);
   obstack_1grow (obs, '\0');
   debug_flush_files ();
   xfprintf (stderr, "%s", (char *) obstack_finish (obs));
@@ -1533,7 +1563,7 @@ m4_errprint (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4___file__ (struct obstack *obs, int argc, token_data **argv)
+m4___file__ (struct obstack *obs, int argc, macro_arguments *argv)
 {
   bad_argc (ARG (0), argc, 0, 0);
   obstack_grow (obs, lquote.string, lquote.length);
@@ -1542,14 +1572,14 @@ m4___file__ (struct obstack *obs, int argc, token_data **argv)
 }
 
 static void
-m4___line__ (struct obstack *obs, int argc, token_data **argv)
+m4___line__ (struct obstack *obs, int argc, macro_arguments *argv)
 {
   bad_argc (ARG (0), argc, 0, 0);
   shipout_int (obs, current_line);
 }
 
 static void
-m4___program__ (struct obstack *obs, int argc, token_data **argv)
+m4___program__ (struct obstack *obs, int argc, macro_arguments *argv)
 {
   bad_argc (ARG (0), argc, 0, 0);
   obstack_grow (obs, lquote.string, lquote.length);
@@ -1567,7 +1597,7 @@ m4___program__ (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_m4exit (struct obstack *obs, int argc, token_data **argv)
+m4_m4exit (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   int exit_code = EXIT_SUCCESS;
@@ -1600,14 +1630,14 @@ m4_m4exit (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_m4wrap (struct obstack *obs, int argc, token_data **argv)
+m4_m4wrap (struct obstack *obs, int argc, macro_arguments *argv)
 {
   if (bad_argc (ARG (0), argc, 1, -1))
     return;
   if (no_gnu_extensions)
     obstack_grow (obs, ARG (1), strlen (ARG (1)));
   else
-    dump_args (obs, argc, argv, " ", false);
+    dump_args (obs, 1, argv, " ", false);
   obstack_1grow (obs, '\0');
   push_wrapup ((char *) obstack_finish (obs));
 }
@@ -1632,7 +1662,7 @@ set_trace (symbol *sym, void *data)
 }
 
 static void
-m4_traceon (struct obstack *obs, int argc, token_data **argv)
+m4_traceon (struct obstack *obs, int argc, macro_arguments *argv)
 {
   symbol *s;
   int i;
@@ -1652,7 +1682,7 @@ m4_traceon (struct obstack *obs, int argc, token_data **argv)
 `------------------------------------------------------------------------*/
 
 static void
-m4_traceoff (struct obstack *obs, int argc, token_data **argv)
+m4_traceoff (struct obstack *obs, int argc, macro_arguments *argv)
 {
   symbol *s;
   int i;
@@ -1675,7 +1705,7 @@ m4_traceoff (struct obstack *obs, int argc, token_data **argv)
 `----------------------------------------------------------------------*/
 
 static void
-m4_debugmode (struct obstack *obs, int argc, token_data **argv)
+m4_debugmode (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   const char *str = ARG (1);
@@ -1727,7 +1757,7 @@ m4_debugmode (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_debugfile (struct obstack *obs, int argc, token_data **argv)
+m4_debugfile (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
 
@@ -1748,7 +1778,7 @@ m4_debugfile (struct obstack *obs, int argc, token_data **argv)
 `---------------------------------------------*/
 
 static void
-m4_len (struct obstack *obs, int argc, token_data **argv)
+m4_len (struct obstack *obs, int argc, macro_arguments *argv)
 {
   if (bad_argc (ARG (0), argc, 1, 1))
     return;
@@ -1761,7 +1791,7 @@ m4_len (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_index (struct obstack *obs, int argc, token_data **argv)
+m4_index (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *haystack;
   const char *needle;
@@ -1801,7 +1831,7 @@ m4_index (struct obstack *obs, int argc, token_data **argv)
 `-------------------------------------------------------------------------*/
 
 static void
-m4_substr (struct obstack *obs, int argc, token_data **argv)
+m4_substr (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   int start = 0;
@@ -1885,7 +1915,7 @@ expand_ranges (const char *s, struct obstack *obs)
 `----------------------------------------------------------------------*/
 
 static void
-m4_translit (struct obstack *obs, int argc, token_data **argv)
+m4_translit (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *data;
   const char *from;
@@ -1950,7 +1980,7 @@ m4_translit (struct obstack *obs, int argc, token_data **argv)
 `--------------------------------------------------------------*/
 
 static void
-m4_format (struct obstack *obs, int argc, token_data **argv)
+m4_format (struct obstack *obs, int argc, macro_arguments *argv)
 {
   if (bad_argc (ARG (0), argc, 1, -1))
     return;
@@ -2047,7 +2077,7 @@ init_pattern_buffer (struct re_pattern_buffer *buf, struct re_registers *regs)
 `------------------------------------------------------------------*/
 
 static void
-m4_regexp (struct obstack *obs, int argc, token_data **argv)
+m4_regexp (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   const char *victim;		/* first argument */
@@ -2117,7 +2147,7 @@ m4_regexp (struct obstack *obs, int argc, token_data **argv)
 `------------------------------------------------------------------*/
 
 static void
-m4_patsubst (struct obstack *obs, int argc, token_data **argv)
+m4_patsubst (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
   const char *victim;		/* first argument */
@@ -2222,7 +2252,7 @@ m4_patsubst (struct obstack *obs, int argc, token_data **argv)
 `--------------------------------------------------------------------*/
 
 void
-m4_placeholder (struct obstack *obs, int argc, token_data **argv)
+m4_placeholder (struct obstack *obs, int argc, macro_arguments *argv)
 {
   m4_warn (0, NULL, _("builtin `%s' requested by frozen file not found"),
 	   ARG (0));
@@ -2238,7 +2268,7 @@ m4_placeholder (struct obstack *obs, int argc, token_data **argv)
 
 void
 expand_user_macro (struct obstack *obs, symbol *sym,
-		   int argc, token_data **argv)
+		   int argc, macro_arguments *argv)
 {
   const char *text;
   int i;
@@ -2263,7 +2293,7 @@ expand_user_macro (struct obstack *obs, symbol *sym,
 	  else
 	    {
 	      for (i = 0; isdigit (to_uchar (*text)); text++)
-		i = i*10 + (*text - '0');
+		i = i * 10 + (*text - '0');
 	    }
 	  if (i < argc)
 	    obstack_grow (obs, ARG (i), strlen (ARG (i)));
@@ -2276,7 +2306,8 @@ expand_user_macro (struct obstack *obs, symbol *sym,
 
 	case '*':		/* all arguments */
 	case '@':		/* ... same, but quoted */
-	  dump_args (obs, argc, argv, ",", *text == '@');
+	  /* TODO push a $@ reference.  */
+	  dump_args (obs, 1, argv, ",", *text == '@');
 	  text++;
 	  break;
 
diff --git a/src/debug.c b/src/debug.c
index c22a482..e5bd280 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -362,7 +362,7 @@ trace_prepre (const char *name, int id)
 `-----------------------------------------------------------------------*/
 
 void
-trace_pre (const char *name, int id, int argc, token_data **argv)
+trace_pre (const char *name, int id, int argc, macro_arguments *argv)
 {
   int i;
   const builtin *bp;
@@ -379,14 +379,14 @@ trace_pre (const char *name, int id, int argc, token_data **argv)
 	  if (i != 1)
 	    trace_format (", ");
 
-	  switch (TOKEN_DATA_TYPE (argv[i]))
+	  switch (TOKEN_DATA_TYPE (argv->array[i - 1]))
 	    {
 	    case TOKEN_TEXT:
-	      trace_format ("%l%S%r", TOKEN_DATA_TEXT (argv[i]));
+	      trace_format ("%l%S%r", TOKEN_DATA_TEXT (argv->array[i - 1]));
 	      break;
 
 	    case TOKEN_FUNC:
-	      bp = find_builtin_by_addr (TOKEN_DATA_FUNC (argv[i]));
+	      bp = find_builtin_by_addr (TOKEN_DATA_FUNC (argv->array[i - 1]));
 	      if (bp == NULL)
 		{
 		  assert (!"trace_pre");
@@ -417,7 +417,7 @@ trace_pre (const char *name, int id, int argc, token_data **argv)
 `-------------------------------------------------------------------*/
 
 void
-trace_post (const char *name, int id, int argc, token_data **argv,
+trace_post (const char *name, int id, int argc, macro_arguments *argv,
 	    const char *expanded)
 {
   if (debug_level & DEBUG_TRACE_CALL)
diff --git a/src/format.c b/src/format.c
index 96ac562..4c2b60a 100644
--- a/src/format.c
+++ b/src/format.c
@@ -27,21 +27,17 @@
 /* Simple varargs substitute.  We assume int and unsigned int are the
    same size; likewise for long and unsigned long.  */
 
-#define ARG_INT(argc, argv) \
-	((argc == 0) ? 0 : \
-	 (--argc, argv++, atoi (TOKEN_DATA_TEXT (argv[-1]))))
+#define ARG_INT(i, argc, argv)						\
+	((i == argc) ? 0 : atoi (TOKEN_DATA_TEXT (argv->array[i++ - 1])))
 
-#define ARG_LONG(argc, argv) \
-	((argc == 0) ? 0 : \
-	 (--argc, argv++, atol (TOKEN_DATA_TEXT (argv[-1]))))
+#define ARG_LONG(i, argc, argv)						\
+	((i == argc) ? 0L : atol (TOKEN_DATA_TEXT (argv->array[i++ - 1])))
 
-#define ARG_STR(argc, argv) \
-	((argc == 0) ? "" : \
-	 (--argc, argv++, TOKEN_DATA_TEXT (argv[-1])))
+#define ARG_STR(i, argc, argv)						\
+	((i == argc) ? "" : TOKEN_DATA_TEXT (argv->array[i++ - 1]))
 
-#define ARG_DOUBLE(argc, argv) \
-	((argc == 0) ? 0 : \
-	 (--argc, argv++, atof (TOKEN_DATA_TEXT (argv[-1]))))
+#define ARG_DOUBLE(i, argc, argv)					\
+	((i == argc) ? 0.0 : atof (TOKEN_DATA_TEXT (argv->array[i++ - 1])))
 
 
 /*------------------------------------------------------------------.
@@ -53,14 +49,15 @@
 `------------------------------------------------------------------*/
 
 void
-format (struct obstack *obs, int argc, token_data **argv)
+format (struct obstack *obs, int argc, macro_arguments *argv)
 {
-  const char *me = TOKEN_DATA_TEXT (argv[0]);
+  const char *me = argv->argv0;
   const char *f;			/* format control string */
   const char *fmt;			/* position within f */
   char fstart[] = "%'+- 0#*.*hhd";	/* current format spec */
   char *p;				/* position within fstart */
   unsigned char c;			/* a simple character */
+  int index = 1;			/* index within argc used so far */
 
   /* Flags.  */
   char flags;				/* flags to use in fstart */
@@ -88,9 +85,7 @@ format (struct obstack *obs, int argc, token_data **argv)
   char *str;			/* malloc'd buffer of formatted text */
   enum {CHAR, INT, LONG, DOUBLE, STR} datatype;
 
-  argv++;
-  argc--;
-  f = fmt = ARG_STR (argc, argv);
+  f = fmt = ARG_STR (index, argc, argv);
   memset (ok, 0, sizeof ok);
   for (;;)
     {
@@ -175,7 +170,7 @@ format (struct obstack *obs, int argc, token_data **argv)
       *p++ = '*';
       if (*fmt == '*')
 	{
-	  width = ARG_INT (argc, argv);
+	  width = ARG_INT (index, argc, argv);
 	  fmt++;
 	}
       else
@@ -195,7 +190,7 @@ format (struct obstack *obs, int argc, token_data **argv)
 	  ok['c'] = 0;
 	  if (*(++fmt) == '*')
 	    {
-	      prec = ARG_INT (argc, argv);
+	      prec = ARG_INT (index, argc, argv);
 	      ++fmt;
 	    }
 	  else
@@ -280,27 +275,27 @@ format (struct obstack *obs, int argc, token_data **argv)
       switch (datatype)
 	{
 	case CHAR:
-	  str = xasprintf (fstart, width, ARG_INT(argc, argv));
+	  str = xasprintf (fstart, width, ARG_INT (index, argc, argv));
 	  break;
 
 	case INT:
-	  str = xasprintf (fstart, width, prec, ARG_INT(argc, argv));
+	  str = xasprintf (fstart, width, prec, ARG_INT (index, argc, argv));
 	  break;
 
 	case LONG:
-	  str = xasprintf (fstart, width, prec, ARG_LONG(argc, argv));
+	  str = xasprintf (fstart, width, prec, ARG_LONG (index, argc, argv));
 	  break;
 
 	case DOUBLE:
-	  str = xasprintf (fstart, width, prec, ARG_DOUBLE(argc, argv));
+	  str = xasprintf (fstart, width, prec, ARG_DOUBLE (index, argc, argv));
 	  break;
 
 	case STR:
-	  str = xasprintf (fstart, width, prec, ARG_STR(argc, argv));
+	  str = xasprintf (fstart, width, prec, ARG_STR (index, argc, argv));
 	  break;
 
 	default:
-	  abort();
+	  abort ();
 	}
 
       /* NULL was returned on failure, such as invalid format string.
diff --git a/src/m4.h b/src/m4.h
index 4f1fa1f..94276e9 100644
--- a/src/m4.h
+++ b/src/m4.h
@@ -89,7 +89,8 @@ typedef struct string STRING;
 
 /* Those must come first.  */
 typedef struct token_data token_data;
-typedef void builtin_func (struct obstack *, int, token_data **);
+typedef struct macro_arguments macro_arguments;
+typedef void builtin_func (struct obstack *, int, macro_arguments *);
 
 /* Gnulib's stdbool doesn't work with bool bitfields.  For nicer
    debugging, use bool when we know it works, but use the more
@@ -103,14 +104,14 @@ typedef unsigned int bool_bitfield;
 /* Take advantage of GNU C compiler source level optimization hints,
    using portable macros.  */
 #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 6)
-#  define M4_GNUC_ATTRIBUTE(args)	__attribute__(args)
+#  define M4_GNUC_ATTRIBUTE(args)	__attribute__ (args)
 #else
 #  define M4_GNUC_ATTRIBUTE(args)
 #endif  /* __GNUC__ */
 
-#define M4_GNUC_UNUSED		M4_GNUC_ATTRIBUTE((__unused__))
+#define M4_GNUC_UNUSED		M4_GNUC_ATTRIBUTE ((__unused__))
 #define M4_GNUC_PRINTF(fmt, arg)			\
-  M4_GNUC_ATTRIBUTE((__format__ (__printf__, fmt, arg)))
+  M4_GNUC_ATTRIBUTE ((__format__ (__printf__, fmt, arg)))
 
 /* File: m4.c  --- global definitions.  */
 
@@ -132,12 +133,13 @@ extern const char *user_word_regexp;	/* -W */
 extern int retcode;
 extern const char *program_name;
 
-void m4_error (int, int, const char *, const char *, ...) M4_GNUC_PRINTF(4, 5);
+void m4_error (int, int, const char *, const char *, ...)
+  M4_GNUC_PRINTF (4, 5);
 void m4_error_at_line (int, int, const char *, int, const char *,
-		       const char *, ...) M4_GNUC_PRINTF(6, 7);
-void m4_warn (int, const char *, const char *, ...) M4_GNUC_PRINTF(3, 4);
+		       const char *, ...) M4_GNUC_PRINTF (6, 7);
+void m4_warn (int, const char *, const char *, ...) M4_GNUC_PRINTF (3, 4);
 void m4_warn_at_line (int, const char *, int, const char *,
-		      const char *, ...) M4_GNUC_PRINTF(5, 6);
+		      const char *, ...) M4_GNUC_PRINTF (5, 6);
 
 #ifdef USE_STACKOVF
 void setup_stackovf_trap (char *const *, char *const *,
@@ -235,11 +237,13 @@ bool debug_set_output (const char *, const char *);
 void debug_message_prefix (void);
 
 void trace_prepre (const char *, int);
-void trace_pre (const char *, int, int, token_data **);
-void trace_post (const char *, int, int, token_data **, const char *);
+void trace_pre (const char *, int, int, macro_arguments *);
+void trace_post (const char *, int, int, macro_arguments *, const char *);
 
 /* File: input.c  --- lexical definitions.  */
 
+typedef struct token_chain token_chain;
+
 /* Various different token types.  */
 enum token_type
 {
@@ -256,9 +260,19 @@ enum token_type
 /* The data for a token, a macro argument, and a macro definition.  */
 enum token_data_type
 {
-  TOKEN_VOID,
-  TOKEN_TEXT,
-  TOKEN_FUNC
+  TOKEN_VOID, /* Token still being constructed, u is invalid.  */
+  TOKEN_TEXT, /* Straight text, u.u_t is valid.  */
+  TOKEN_FUNC, /* Builtin function definition, u.func is valid.  */
+  TOKEN_COMP  /* Composite argument, u.chain is valid.  */
+};
+
+/* Composite tokens are built of a linked list of chains.  */
+struct token_chain
+{
+  token_chain *next; /* Pointer to next link of chain.  */
+  char *str; /* NUL-terminated string if text, else NULL.  */
+  macro_arguments *argv; /* Reference to earlier $@.  */
+  unsigned int index; /* Index within argv to start reading from.  */
 };
 
 struct token_data
@@ -275,10 +289,31 @@ struct token_data
 	}
       u_t;
       builtin_func *func;
+
+      /* Composite text: a linked list of straight text and $@
+	 placeholders.  */
+      token_chain *chain;
     }
   u;
 };
 
+struct macro_arguments
+{
+  /* Number of arguments owned by this object, may be larger than
+     arraylen since the array can refer to multiple arguments via a
+     single $@ reference.  */
+  unsigned int argc;
+  /* False unless the macro expansion refers to $@, determines whether
+     this object can be freed at end of macro expansion or must wait
+     until next byte read from file.  */
+  bool inuse;
+  const char *argv0; /* The macro name being expanded.  */
+  size_t arraylen; /* True length of allocated elements in array.  */
+  /* Used as a variable-length array, storing information about each
+     argument.  */
+  token_data *array[FLEXIBLE_ARRAY_MEMBER];
+};
+
 #define TOKEN_DATA_TYPE(Td)		((Td)->type)
 #define TOKEN_DATA_TEXT(Td)		((Td)->u.u_t.text)
 #ifdef ENABLE_CHANGEWORD
@@ -358,7 +393,7 @@ struct symbol
   int pending_expansions;
 
   char *name;
-  token_data data;
+  token_data data;  /* Type should be only TOKEN_TEXT or TOKEN_FUNC.  */
 };
 
 #define SYMBOL_NEXT(S)		((S)->next)
@@ -391,7 +426,7 @@ void hack_all_symbols (hack_symbol *, void *);
 extern int expansion_level;
 
 void expand_input (void);
-void call_macro (symbol *, int, token_data **, struct obstack *);
+void call_macro (symbol *, int, macro_arguments *, struct obstack *);
 
 /* File: builtin.c  --- builtins.  */
 
@@ -427,8 +462,8 @@ void set_macro_sequence (const char *);
 void free_regex (void);
 void define_user_macro (const char *, const char *, symbol_lookup);
 void undivert_all (void);
-void expand_user_macro (struct obstack *, symbol *, int, token_data **);
-void m4_placeholder (struct obstack *, int, token_data **);
+void expand_user_macro (struct obstack *, symbol *, int, macro_arguments *);
+void m4_placeholder (struct obstack *, int, macro_arguments *);
 void init_pattern_buffer (struct re_pattern_buffer *, struct re_registers *);
 const char *ntoa (int32_t, int);
 
@@ -448,7 +483,7 @@ bool evaluate (const char *, const char *, int32_t *);
 
 /* File: format.c  --- printf like formatting.  */
 
-void format (struct obstack *, int, token_data **);
+void format (struct obstack *, int, macro_arguments *);
 
 /* File: freeze.c --- frozen state files.  */
 
diff --git a/src/macro.c b/src/macro.c
index 8a678d4..d2f2cb7 100644
--- a/src/macro.c
+++ b/src/macro.c
@@ -241,19 +241,22 @@ expand_argument (struct obstack *obs, token_data *argp, const char *caller)
 | on the obstack ARGPTR.						   |
 `-------------------------------------------------------------------------*/
 
-static void
-collect_arguments (symbol *sym, struct obstack *argptr,
+static macro_arguments *
+collect_arguments (symbol *sym, struct obstack *argptr, unsigned argv_base,
 		   struct obstack *arguments)
 {
   token_data td;
   token_data *tdp;
   bool more_args;
   bool groks_macro_args = SYMBOL_MACRO_ARGS (sym);
+  macro_arguments args;
+  macro_arguments *argv;
 
-  TOKEN_DATA_TYPE (&td) = TOKEN_TEXT;
-  TOKEN_DATA_TEXT (&td) = SYMBOL_NAME (sym);
-  tdp = (token_data *) obstack_copy (arguments, &td, sizeof td);
-  obstack_ptr_grow (argptr, tdp);
+  args.argc = 1;
+  args.inuse = false;
+  args.argv0 = SYMBOL_NAME (sym);
+  args.arraylen = 0;
+  obstack_grow (argptr, &args, offsetof (macro_arguments, array));
 
   if (peek_token () == TOKEN_OPEN)
     {
@@ -269,9 +272,15 @@ collect_arguments (symbol *sym, struct obstack *argptr,
 	    }
 	  tdp = (token_data *) obstack_copy (arguments, &td, sizeof td);
 	  obstack_ptr_grow (argptr, tdp);
+	  args.arraylen++;
+	  args.argc++;
 	}
       while (more_args);
     }
+  argv = (macro_arguments *) ((char *) obstack_base (argptr) + argv_base);
+  argv->argc = args.argc;
+  argv->arraylen = args.arraylen;
+  return argv;
 }
 
 
@@ -285,13 +294,13 @@ collect_arguments (symbol *sym, struct obstack *argptr,
 `------------------------------------------------------------------------*/
 
 void
-call_macro (symbol *sym, int argc, token_data **argv,
-		 struct obstack *expansion)
+call_macro (symbol *sym, int argc, macro_arguments *argv,
+	    struct obstack *expansion)
 {
   switch (SYMBOL_TYPE (sym))
     {
     case TOKEN_FUNC:
-      (*SYMBOL_FUNC (sym)) (expansion, argc, argv);
+      SYMBOL_FUNC (sym) (expansion, argc, argv);
       break;
 
     case TOKEN_TEXT:
@@ -319,8 +328,8 @@ expand_macro (symbol *sym)
 {
   struct obstack arguments;	/* Alternate obstack if argc_stack is busy.  */
   unsigned argv_base;		/* Size of argv_stack on entry.  */
-  bool use_argc_stack = true;	/* Whether argc_stack is safe.  */
-  token_data **argv;
+  void *argc_start;		/* Start of argc_stack, else NULL if unsafe.  */
+  macro_arguments *argv;
   int argc;
   struct obstack *expansion;
   const char *expanded;
@@ -357,18 +366,17 @@ expand_macro (symbol *sym)
 	 outer invocation has an unfinished argument being
 	 collected.  */
       obstack_init (&arguments);
-      use_argc_stack = false;
+      argc_start = NULL;
     }
+  else
+    argc_start = obstack_finish (&argc_stack);
 
   if (traced && (debug_level & DEBUG_TRACE_CALL))
     trace_prepre (SYMBOL_NAME (sym), my_call_id);
 
-  collect_arguments (sym, &argv_stack,
-		     use_argc_stack ? &argc_stack : &arguments);
-
-  argc = ((obstack_object_size (&argv_stack) - argv_base)
-	  / sizeof (token_data *));
-  argv = (token_data **) ((char *) obstack_base (&argv_stack) + argv_base);
+  argv = collect_arguments (sym, &argv_stack, argv_base,
+			    argc_start ? &argc_stack : &arguments);
+  argc = argv->argc;
 
   loc_close_file = current_file;
   loc_close_line = current_line;
@@ -394,9 +402,10 @@ expand_macro (symbol *sym)
   if (SYMBOL_DELETED (sym))
     free_symbol (sym);
 
-  if (use_argc_stack)
-    obstack_free (&argc_stack, argv[0]);
+  /* TODO pay attention to argv->inuse, in case someone is depending on $@.  */
+  if (argc_start)
+    obstack_free (&argc_stack, argc_start);
   else
     obstack_free (&arguments, NULL);
-  obstack_blank (&argv_stack, -argc * sizeof (token_data *));
+  obstack_blank (&argv_stack, argv_base - obstack_object_size (&argv_stack));
 }
-- 
1.5.3.5