Patch for PGunescapeBytea

Ben Lamb <[email protected]>
Newsgroups gmane.comp.python.db.pypgsql.user,gmane.comp.db.postgresql.devel.patches
Message-ID <[email protected]>
Hi,

I found the libpq function PGunescapeBytea a little slow. It was taking a 
minute and a half to decode a 500Kb on a fairly fast machine. I think the 
culprit is sscanf.

I attach a patch that replaces the function with one used to perform the same 
task in pyPgSQL (a Python interface to PostgreSQL). This code was written by 
Billy Allie, author of pyPgSQL. I've changed a few variable names to match 
those in the original code and removed a bit of Pythonness.

I've checked with the author, the code is licensed under a BSD license.

I've performed limited testing of the function by putting JPEGs into 
PostgreSQL, extracting them using them using the new function and diffing 
against the original files.

The new function is significantly faster on my machine with the JPEGs being 
decoded in less than a second. I attach a modified libpq example program that 
I used for my testing.

Regards,

Ben Lamb.

The patch modifies fe-exec.c in /src/interfaces/libpq from PostgreSQL 7.3.2.
fe-exec.c.patch (text/x-diff, 2.7 KB)
--- fe-exec.c	Wed Apr 30 17:26:49 2003
+++ fe-exec.c.new	Thu May  1 19:41:38 2003
@@ -40,6 +40,7 @@
 };
 
 
+#define VAL(CH)  ((CH) - '0')
 /* Note: DONOTICE macro will work if applied to either PGconn or PGresult */
 #define DONOTICE(conn,message) \
 	((*(conn)->noticeHook) ((conn)->noticeArg, (message)))
@@ -186,29 +187,13 @@
  *		pointer to the buffer which is NULL on error, and the size of the
  *		buffer in retbuflen. The pointer may subsequently be used as an
  *		argument to the function free(3). It is the reverse of PQescapeBytea.
- *
- *		The following transformations are reversed:
- *		'\0' == ASCII  0 == \000
- *		'\'' == ASCII 39 == \'
- *		'\\' == ASCII 92 == \\
- *
- *		States:
- *		0	normal		0->1->2->3->4
- *		1	\			   1->5
- *		2	\0			   1->6
- *		3	\00
- *		4	\000
- *		5	\'
- *		6	\\
  */
 unsigned char *
 PQunescapeBytea(unsigned char *strtext, size_t *retbuflen)
 {
 	size_t		buflen;
-	unsigned char *buffer,
-			   *sp,
-			   *bp;
-	unsigned int state = 0;
+	unsigned char *buffer;
+	int i, j, byte;
 
 	if (strtext == NULL)
 		return NULL;
@@ -217,70 +202,40 @@
 	buffer = (unsigned char *) malloc(buflen);	/* isn't NULL terminated */
 	if (buffer == NULL)
 		return NULL;
-	for (bp = buffer, sp = strtext; *sp != '\0'; bp++, sp++)
+	
+	for (i = j = 0; i < buflen;)
 	{
-		switch (state)
+
+		switch (strtext[i])
 		{
-			case 0:
-				if (*sp == '\\')
-					state = 1;
-				*bp = *sp;
-				break;
-			case 1:
-				if (*sp == '\'')	/* state=5 */
-				{				/* replace \' with 39 */
-					bp--;
-					*bp = '\'';
-					buflen--;
-					state = 0;
-				}
-				else if (*sp == '\\')	/* state=6 */
-				{				/* replace \\ with 92 */
-					bp--;
-					*bp = '\\';
-					buflen--;
-					state = 0;
-				}
+			case '\\':
+				i++;
+				if (strtext[i] == '\\')
+					buffer[j++] = strtext[i++];
 				else
 				{
-					if (isdigit(*sp))
-						state = 2;
-					else
-						state = 0;
-					*bp = *sp;
+					if ((!isdigit(strtext[i])) ||
+					    (!isdigit(strtext[i+1])) ||
+					    (!isdigit(strtext[i+2])))
+						return NULL;
+
+					byte = VAL(strtext[i++]);
+					byte = (byte << 3) + VAL(strtext[i++]);	
+					buffer[j++] = (byte << 3) + VAL(strtext[i++]);
 				}
 				break;
-			case 2:
-				if (isdigit(*sp))
-					state = 3;
-				else
-					state = 0;
-				*bp = *sp;
-				break;
-			case 3:
-				if (isdigit(*sp))		/* state=4 */
-				{
-					int			v;
 
-					bp -= 3;
-					sscanf(sp - 2, "%03o", &v);
-					*bp = v;
-					buflen -= 3;
-					state = 0;
-				}
-				else
-				{
-					*bp = *sp;
-					state = 0;
-				}
-				break;
+			default:
+				buffer[j++] = strtext[i++];
 		}
 	}
-	buffer = realloc(buffer, buflen);
+
+	buffer[j] = (char)0;
+	buffer = realloc(buffer, j);
 	if (buffer == NULL)
 		return NULL;
 
-	*retbuflen = buflen;
+	*retbuflen = j;
 	return buffer;
 }
testlibpq.c (text/x-csrc, 3.3 KB)
/*
 * testlibpq.c
 *		Test the C version of LIBPQ, the POSTGRES frontend library.
 *
 *
 */
#include <stdio.h>
#include "libpq-fe.h"

static void
exit_nicely(PGconn *conn)
{
	PQfinish(conn);
	exit(1);
}

int
main()
{
	char	   *pghost,
			   *pgport,
			   *pgoptions,
			   *pgtty;
	char	   *dbName;
	int			nFields;
	int			i,
				j;

	FILE	*image;
#ifdef DEBUG
	FILE	   *debug;
#endif   /* DEBUG */

	PGconn	   *conn;
	PGresult   *res;

	/*
	 * begin, by setting the parameters for a backend connection if the
	 * parameters are null, then the system will try to use reasonable
	 * defaults by looking up environment variables or, failing that,
	 * using hardwired constants
	 */
	pghost = NULL;				/* host name of the backend server */
	pgport = NULL;				/* port of the backend server */
	pgoptions = NULL;			/* special options to start up the backend
								 * server */
	pgtty = NULL;				/* debugging tty for the backend server */
	dbName = "ben";

	/* make a connection to the database */
	conn = PQsetdb(pghost, pgport, pgoptions, pgtty, dbName);

	/* check to see that the backend connection was successfully made */
	if (PQstatus(conn) == CONNECTION_BAD)
	{
		fprintf(stderr, "Connection to database '%s' failed.\n", dbName);
		fprintf(stderr, "%s", PQerrorMessage(conn));
		exit_nicely(conn);
	}

#ifdef DEBUG
	debug = fopen("/tmp/trace.out", "w");
	PQtrace(conn, debug);
#endif   /* DEBUG */

	/* start a transaction block */
	res = PQexec(conn, "BEGIN");
	if (PQresultStatus(res) != PGRES_COMMAND_OK)
	{
		fprintf(stderr, "BEGIN command failed\n");
		PQclear(res);
		exit_nicely(conn);
	}

	/*
	 * should PQclear PGresult whenever it is no longer needed to avoid
	 * memory leaks
	 */
	PQclear(res);

	/*
	 * fetch instances from the pg_database, the system catalog of
	 * databases
	 */
	res = PQexec(conn, "DECLARE myportal CURSOR FOR select picdata, location from est_houses WHERE id = 81");
	if (PQresultStatus(res) != PGRES_COMMAND_OK)
	{
		fprintf(stderr, "DECLARE CURSOR command failed\n");
		PQclear(res);
		exit_nicely(conn);
	}
	PQclear(res);

	res = PQexec(conn, "FETCH ALL in myportal");
	if (PQresultStatus(res) != PGRES_TUPLES_OK)
	{
		fprintf(stderr, "FETCH ALL command didn't return tuples properly\n");
		PQclear(res);
		exit_nicely(conn);
	}

	/* first, print out the attribute names */
	nFields = PQnfields(res);
	for (i = 0; i < nFields; i++)
		printf("%-15s", PQfname(res, i));
	printf("\n\n");

	/* next, print out the instances */
	for (i = 0; i < PQntuples(res); i++)
	{
		/* Write image to disk */
		char* bytea = PQgetvalue(res, i, 0);
		unsigned char* bytea_output;
		size_t bytea_size;
		printf("Attempting to decode bytea data...\n");
	        bytea_output = PQunescapeBytea((unsigned char*)bytea, &bytea_size);
		if (bytea_output != NULL)
		{
			printf("Writing image to disk...\n");
			image = fopen("test.jpg", "w");
			fwrite(bytea_output, bytea_size, 1, image);
			fclose(image);
			printf("Written image...\n");
			free(bytea_output, bytea_size);
		}

		for (j = 1; j < nFields; j++)
			printf("%-15s", PQgetvalue(res, i, j));
		printf("\n");
	}

	PQclear(res);

	/* close the portal */
	res = PQexec(conn, "CLOSE myportal");
	PQclear(res);

	/* end the transaction */
	res = PQexec(conn, "END");
	PQclear(res);

	/* close the connection to the database and cleanup */
	PQfinish(conn);

#ifdef DEBUG
	fclose(debug);
#endif   /* DEBUG */

	return 0;
}
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.