mdb_map_find_next returning wrong pages

Nirgal Vourgère <[email protected]> Tue, 12 Oct 2010 18:34:55 +0200
Newsgroups gmane.comp.db.mdb-tools.devel
Message-ID <[email protected]>
I found a problem with mdb-export SEGFAULTing on a big table. This is JET3 engine.

CREATE TABLE "DDLogDr"
 (
    "BatchId"           INTEGER,
    "Date"          TIMESTAMP WITHOUT TIME ZONE,
    "iTransactionID"            INTEGER,
    "iConstituentID"            INTEGER,
    "cAmount"           NUMERIC(15,2),
    "dDate"         TIMESTAMP WITHOUT TIME ZONE,
    "sAuthorisation"            VARCHAR (50),
    "iAuthorisationStatusID"            INTEGER,
    "sBankAccountName"          VARCHAR (50),
    "sBankAccountNumber"            VARCHAR (30),
    "iAppealID"         INTEGER
);

Here's the end of output:
Row 0 bytes 8 to 2047  [delflag]
Row 1 bytes 2376 to 7 [lookup] [delflag]
Row 2 bytes 5662 to 2375

Program received signal SIGSEGV, Segmentation fault.
0xb7fdae99 in mdb_crack_row3 (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:121
121         var_col_offsets[i] = mdb->pg_buf[col_ptr-i]+(jumps_used*256);
(gdb) info program
    Using the running image of child process 17636.
Program stopped at 0xb7fdae99.
It stopped with signal SIGSEGV, Segmentation fault.
(gdb) bt
#0  0xb7fdae99 in mdb_crack_row3 (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:121
#1  mdb_crack_row (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:185
#2  0xb7fd6595 in mdb_read_row (table=0x805f300, row=2) at data.c:277
#3  0xb7fd689a in mdb_fetch_row (table=0x805f300) at data.c:411
#4  0x080492d2 in main (argc=3, argv=0xbffff3a4) at mdb-export.c:193
(gdb) info locals
i = 134533134
col_ptr = <value optimized out>
num_jumps = 16777202
jumps_used = 295
(gdb) up
#1  mdb_crack_row (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:185
185             mdb_crack_row3(mdb, row_start, row_end, bitmask_sz,
(gdb) info locals
col = <value optimized out>
mdb = 0x804c800
row_var_cols = 0
row_cols = 255
nullmask = 0x804d138 ""
bitmask_sz = 32
fixed_cols_found = 134531072
row_fixed_cols = 2342
col_count_size = 1
i = <value optimized out>

Defining SLOW_READ in src/libmdb/data.c fixes the problem.
So I traced back the problem to a incorrect page being returned by mdb_map_find_next.

I noticed that the slow version is testing if the page is correct with byte #0 if the page being 1 (MDB_PAGE_DATA) and bytes #4..7 matching entry->table_pg.
So I simply added that test in the not-slow version. This works ok. I get the good record count (4,170,473) in my problematic case.

I was a bit confused by the fact that function mdb_map_find_next returns -1 on failure, but that the return type was guint32.
This results in the test in mdb_read_next_dpg that produce "Warning: defaulting to brute force read" never being called.

I order to fix that, I had to change the type returned by mdb_map_find_next from guint32 to gint32. This implies another change in mdb_map_find_next_freepage() and in src/util/prfreemap.c.

Attached is a patch, based on "my" debian-based version. http://nirgal.com/mdbtools
Hopefully, it should also applies on various trunk/master out there...

------------------------------------------------------------------------------
Beautiful is writing same markup. Internet Explorer 9 supports
standards for HTML5, CSS3, SVG 1.1,  ECMAScript5, and DOM L2 & L3.
Spend less time writing and  rewriting code and more time creating great
experiences on the web. Be a part of the beta today.
http://p.sf.net/sfu/beautyoftheweb

_______________________________________________
mdbtools-dev mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/mdbtools-dev
180-slowmap-fallback (text/x-patch, 4.6 KB)
Index: mdbtools-0.6pre1/src/libmdb/data.c
===================================================================
--- mdbtools-0.6pre1.orig/src/libmdb/data.c
+++ mdbtools-0.6pre1/src/libmdb/data.c
@@ -256,7 +256,10 @@
 	if (table->num_rows == 0) 
 		return 0;
 
-	mdb_find_row(mdb, row, &row_start, &row_size);
+	if (mdb_find_row(mdb, row, &row_start, &row_size)) {
+		fprintf(stderr, "warning: mdb_find_row failed.");
+		return 0;
+	}
 
 	delflag = lookupflag = 0;
 	if (row_start & 0x8000) lookupflag++;
@@ -315,6 +318,8 @@
 	}
 	return 1;
 }
+
+/* Read next data page into mdb->pg_buf */
 int mdb_read_next_dpg(MdbTableDef *table)
 {
 	MdbCatalogEntry *entry = table->entry;
@@ -322,16 +327,28 @@
 	int next_pg;
 
 #ifndef SLOW_READ
-	next_pg = mdb_map_find_next(mdb, table->usage_map,
-		table->map_sz, table->cur_phys_pg);
+	while (1) {
+		next_pg = mdb_map_find_next(mdb, table->usage_map,
+			table->map_sz, table->cur_phys_pg);
+		if (next_pg < 0)
+			break; /* unknow map type: goto fallback */
+		if (!next_pg)
+			return 0;
 
-	if (next_pg >= 0) {
-		if (mdb_read_pg(mdb, next_pg)) {
-			table->cur_phys_pg = next_pg;
-			return table->cur_phys_pg;
-		} else {
+		if (!mdb_read_pg(mdb, next_pg)) {
+			fprintf(stderr, "error: reading page %d failed.\n", next_pg);
 			return 0;
 		}
+
+		table->cur_phys_pg = next_pg;
+		if (mdb->pg_buf[0]==MDB_PAGE_DATA && mdb_get_int32(mdb->pg_buf, 4)==entry->table_pg)
+			return table->cur_phys_pg;
+
+		/* On rare occasion, mdb_map_find_next will return a wrong page */
+		/* Found in a big file, over 4,000,000 records */
+		fprintf(stderr,
+			"warning: page %d from map doesn't match: Type=%d, buf[4..7]=%d Expected table_pg=%d\n",
+			next_pg, mdb_get_int32(mdb->pg_buf, 4), entry->table_pg);
 	}
 	fprintf(stderr, "Warning: defaulting to brute force read\n");
 #endif 
@@ -339,7 +356,7 @@
 	do {
 		if (!mdb_read_pg(mdb, table->cur_phys_pg++))
 			return 0;
-	} while (mdb->pg_buf[0]!=0x01 || mdb_get_int32(mdb->pg_buf, 4)!=entry->table_pg);
+	} while (mdb->pg_buf[0]!=MDB_PAGE_DATA || mdb_get_int32(mdb->pg_buf, 4)!=entry->table_pg);
 	/* fprintf(stderr,"returning new page %ld\n", table->cur_phys_pg); */
 	return table->cur_phys_pg;
 }
@@ -396,7 +413,7 @@
 		} else {
 			rows = mdb_get_int16(mdb->pg_buf,fmt->row_count_offset);
 
-			/* if at end of page, find a new page */
+			/* if at end of page, find a new data page */
 			if (table->cur_row >= rows) {
 				table->cur_row=0;
 	
Index: mdbtools-0.6pre1/include/mdbtools.h
===================================================================
--- mdbtools-0.6pre1.orig/include/mdbtools.h
+++ mdbtools-0.6pre1/include/mdbtools.h
@@ -511,7 +511,7 @@
 
 /* map.c */
 extern guint32 mdb_map_find_next_freepage(MdbTableDef *table, int row_size);
-extern guint32 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg);
+extern gint32 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg);
 
 /* props.c */
 extern GPtrArray *mdb_read_props_list(gchar *kkd, int len);
Index: mdbtools-0.6pre1/src/libmdb/map.c
===================================================================
--- mdbtools-0.6pre1.orig/src/libmdb/map.c
+++ mdbtools-0.6pre1/src/libmdb/map.c
@@ -23,7 +23,7 @@
 #include "dmalloc.h"
 #endif
 
-static guint32 
+static gint32
 mdb_map_find_next0(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
 {
 	guint32 pgnum, i, usage_bitlen;
@@ -42,7 +42,7 @@
 	/* didn't find anything */
 	return 0;
 }
-static int 
+static gint32
 mdb_map_find_next1(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
 {
 	guint32 map_ind, max_map_pgs, offset, usage_bitlen;
@@ -83,7 +83,10 @@
 	/* didn't find anything */
 	return 0;
 }
-guint32 
+
+/* returns 0 on EOF */
+/* returns -1 on error (unsupported map type) */
+gint32
 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
 {
 	if (map[0] == 0) {
@@ -119,6 +122,9 @@
 			/* allocate new page */
 			pgnum = mdb_alloc_page(table);
 			return pgnum;
+		} else if (pgnum==-1) {
+			fprintf(stderr, "Error: mdb_map_find_next_freepage error while reading maps.\n");
+			exit(1);
 		}
 		cur_pg = pgnum;
 
Index: mdbtools-0.6pre1/src/util/prfreemap.c
===================================================================
--- mdbtools-0.6pre1.orig/src/util/prfreemap.c
+++ mdbtools-0.6pre1/src/util/prfreemap.c
@@ -48,7 +48,7 @@
 	if (map_buf[0]==0)
 		while (map_buf[map_sz]==0xff) map_sz--;
 
-	while (pgnum = mdb_map_find_next(mdb, map_buf, map_sz, pgnum)) {
+	while ((pgnum = mdb_map_find_next(mdb, map_buf, map_sz, pgnum))>0) {
 		printf("%6lu ",(long unsigned) pgnum);
 		if (coln==10) {
 			printf("\n");