[PATCH 2/6] simplify the inlined side of nextchar()

* make sure that data stream->buffer + stream->size always points to
'\0'.  That is enough to send nextchar() towards the slow path without
the need to check offset for overflow.
* replace stream->offset with stream->current - pointer to current
location in buffer rather than offset in it.
* have the increments of stream->current and stream->pos done before
we check whether we need to call nextchar_slow() (with nextchar_slow()
adjusted to be called with incremented ->current and ->pos).

Signed-off-by: Al Viro <[email protected]>
---
 tokenize.c | 71 +++++++++++++++++++++++++++---------------------------
 1 file changed, 36 insertions(+), 35 deletions(-)

diff --git a/tokenize.c b/tokenize.c
index c3c6c234..7c12cf6e 100644
--- a/tokenize.c
+++ b/tokenize.c
@@ -47,7 +47,8 @@ unsigned int tabstop = 8;
 #define BUFSIZE (8192)
 
 typedef struct {
-	int fd, offset, size;
+	unsigned char *current;
+	int fd, size;
 	int pos, line, nr;
 	int newline, whitespace;
 	struct token **tokenlist;
@@ -351,31 +352,34 @@ static struct token * alloc_token(stream_t *stream)
  */
 static int nextchar_slow(stream_t *stream)
 {
-	int offset = stream->offset;
+	unsigned char *p = stream->current;	// bumped by fast path
 	int size = stream->size;
 	int c;
-	int spliced = 0, had_cr, had_backslash;
+	bool spliced = false, had_cr, had_backslash;
 
 restart:
-	had_cr = had_backslash = 0;
+	had_cr = had_backslash = false;
 
 repeat:
-	if (offset >= size) {
+	if (p > stream->buffer + size) {
 		if (stream->fd < 0)
 			goto got_eof;
 		size = read(stream->fd, stream->buffer, BUFSIZE);
 		if (size <= 0)
 			goto got_eof;
+		stream->buffer[size] = '\0';	// sentry
 		stream->size = size;
-		stream->offset = offset = 0;
+		stream->current = stream->buffer;
+		p = stream->buffer + 1;
 	}
 
-	c = stream->buffer[offset++];
+	c = p[-1];
 	if (had_cr)
 		goto check_lf;
 
 	if (c == '\r') {
-		had_cr = 1;
+		had_cr = true;
+		p++;
 		goto repeat;
 	}
 
@@ -383,6 +387,7 @@ norm:
 	if (!had_backslash) {
 		switch (c) {
 		case '\t':
+			stream->pos--;
 			stream->pos += tabstop - stream->pos % tabstop;
 			break;
 		case '\n':
@@ -391,38 +396,40 @@ norm:
 			stream->newline = 1;
 			break;
 		case '\\':
-			had_backslash = 1;
+			had_backslash = true;
 			stream->pos++;
+			p++;
 			goto repeat;
-		default:
-			stream->pos++;
 		}
 	} else {
 		if (c == '\n') {
 			stream->line++;
-			stream->pos = 0;
-			spliced = 1;
+			stream->pos = 1;
+			spliced = true;
+			p++;
 			goto restart;
 		}
-		offset--;
 		c = '\\';
+		stream->pos--;
+		p--;
 	}
-out:
-	stream->offset = offset;
+	stream->current = p;
 
 	return c;
 
-check_lf:
+check_lf:	// CR+LF => LF, solitary CR => LF
 	if (c != '\n')
-		offset--;
+		p--;
 	c = '\n';
 	goto norm;
 
 got_eof:
-	if (had_backslash) {
-		c = '\\';
-		goto out;
-	}
+	stream->pos--;
+	stream->buffer[0] = '\0';	// sentry
+	stream->current = stream->buffer;
+	stream->size = 0;
+	if (had_backslash)
+		return '\\';
 	if (stream->pos & Wnewline_eof)
 		warning(stream_pos(stream), "no newline at end of file");
 	else if (spliced)
@@ -437,16 +444,10 @@ got_eof:
  */
 static inline int nextchar(stream_t *stream)
 {
-	int offset = stream->offset;
-
-	if (offset < stream->size) {
-		int c = stream->buffer[offset++];
-		if (c >= ' ' && c != '\\') {
-			stream->offset = offset;
-			stream->pos++;
-			return c;
-		}
-	}
+	int c = *stream->current++;
+	stream->pos++;
+	if (c != '\\' && c >= ' ')
+		return c;
 	return nextchar_slow(stream);
 }
 
@@ -972,9 +973,8 @@ static struct token *setup_stream(stream_t *stream, int idx, int fd,
 
 	stream->token = NULL;
 	stream->fd = fd;
-	stream->offset = 0;
 	stream->size = buf_size;
-	stream->buffer = buf;
+	stream->current = stream->buffer = buf;
 
 	begin = alloc_token(stream);
 	token_type(begin) = TOKEN_STREAMBEGIN;
@@ -1014,7 +1014,7 @@ struct token * tokenize(const struct position *pos, const char *name, int fd, st
 {
 	struct token *begin, *end;
 	stream_t stream;
-	unsigned char buffer[BUFSIZE];
+	unsigned char buffer[BUFSIZE + 1];
 	int idx;
 
 	idx = init_stream(pos, name, fd, next_path);
@@ -1023,6 +1023,7 @@ struct token * tokenize(const struct position *pos, const char *name, int fd, st
 		return endtoken;
 	}
 
+	buffer[0] = '\0';
 	begin = setup_stream(&stream, idx, fd, buffer, 0);
 	end = tokenize_stream(&stream);
 	if (endtoken)
-- 
2.47.3