mdb_map_find_next returning wrong pages
Nirgal Vourgère <[email protected]> Tue, 12 Oct 2010 18:34:55 +0200
| Newsgroups | gmane.comp.db.mdb-tools.devel |
|---|---|
| Message-ID | <[email protected]> |
I found a problem with mdb-export SEGFAULTing on a big table. This is JET3 engine.
CREATE TABLE "DDLogDr"
(
"BatchId" INTEGER,
"Date" TIMESTAMP WITHOUT TIME ZONE,
"iTransactionID" INTEGER,
"iConstituentID" INTEGER,
"cAmount" NUMERIC(15,2),
"dDate" TIMESTAMP WITHOUT TIME ZONE,
"sAuthorisation" VARCHAR (50),
"iAuthorisationStatusID" INTEGER,
"sBankAccountName" VARCHAR (50),
"sBankAccountNumber" VARCHAR (30),
"iAppealID" INTEGER
);
Here's the end of output:
Row 0 bytes 8 to 2047 [delflag]
Row 1 bytes 2376 to 7 [lookup] [delflag]
Row 2 bytes 5662 to 2375
Program received signal SIGSEGV, Segmentation fault.
0xb7fdae99 in mdb_crack_row3 (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:121
121 var_col_offsets[i] = mdb->pg_buf[col_ptr-i]+(jumps_used*256);
(gdb) info program
Using the running image of child process 17636.
Program stopped at 0xb7fdae99.
It stopped with signal SIGSEGV, Segmentation fault.
(gdb) bt
#0 0xb7fdae99 in mdb_crack_row3 (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:121
#1 mdb_crack_row (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:185
#2 0xb7fd6595 in mdb_read_row (table=0x805f300, row=2) at data.c:277
#3 0xb7fd689a in mdb_fetch_row (table=0x805f300) at data.c:411
#4 0x080492d2 in main (argc=3, argv=0xbffff3a4) at mdb-export.c:193
(gdb) info locals
i = 134533134
col_ptr = <value optimized out>
num_jumps = 16777202
jumps_used = 295
(gdb) up
#1 mdb_crack_row (table=0x805f300, row_start=5662, row_end=2375, fields=0xbfffd9f8) at write.c:185
185 mdb_crack_row3(mdb, row_start, row_end, bitmask_sz,
(gdb) info locals
col = <value optimized out>
mdb = 0x804c800
row_var_cols = 0
row_cols = 255
nullmask = 0x804d138 ""
bitmask_sz = 32
fixed_cols_found = 134531072
row_fixed_cols = 2342
col_count_size = 1
i = <value optimized out>
Defining SLOW_READ in src/libmdb/data.c fixes the problem.
So I traced back the problem to a incorrect page being returned by mdb_map_find_next.
I noticed that the slow version is testing if the page is correct with byte #0 if the page being 1 (MDB_PAGE_DATA) and bytes #4..7 matching entry->table_pg.
So I simply added that test in the not-slow version. This works ok. I get the good record count (4,170,473) in my problematic case.
I was a bit confused by the fact that function mdb_map_find_next returns -1 on failure, but that the return type was guint32.
This results in the test in mdb_read_next_dpg that produce "Warning: defaulting to brute force read" never being called.
I order to fix that, I had to change the type returned by mdb_map_find_next from guint32 to gint32. This implies another change in mdb_map_find_next_freepage() and in src/util/prfreemap.c.
Attached is a patch, based on "my" debian-based version. http://nirgal.com/mdbtools
Hopefully, it should also applies on various trunk/master out there...
------------------------------------------------------------------------------
Beautiful is writing same markup. Internet Explorer 9 supports
standards for HTML5, CSS3, SVG 1.1, ECMAScript5, and DOM L2 & L3.
Spend less time writing and rewriting code and more time creating great
experiences on the web. Be a part of the beta today.
http://p.sf.net/sfu/beautyoftheweb
_______________________________________________
mdbtools-dev mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/mdbtools-dev
180-slowmap-fallback
(text/x-patch, 4.6 KB)
Index: mdbtools-0.6pre1/src/libmdb/data.c
===================================================================
--- mdbtools-0.6pre1.orig/src/libmdb/data.c
+++ mdbtools-0.6pre1/src/libmdb/data.c
@@ -256,7 +256,10 @@
if (table->num_rows == 0)
return 0;
- mdb_find_row(mdb, row, &row_start, &row_size);
+ if (mdb_find_row(mdb, row, &row_start, &row_size)) {
+ fprintf(stderr, "warning: mdb_find_row failed.");
+ return 0;
+ }
delflag = lookupflag = 0;
if (row_start & 0x8000) lookupflag++;
@@ -315,6 +318,8 @@
}
return 1;
}
+
+/* Read next data page into mdb->pg_buf */
int mdb_read_next_dpg(MdbTableDef *table)
{
MdbCatalogEntry *entry = table->entry;
@@ -322,16 +327,28 @@
int next_pg;
#ifndef SLOW_READ
- next_pg = mdb_map_find_next(mdb, table->usage_map,
- table->map_sz, table->cur_phys_pg);
+ while (1) {
+ next_pg = mdb_map_find_next(mdb, table->usage_map,
+ table->map_sz, table->cur_phys_pg);
+ if (next_pg < 0)
+ break; /* unknow map type: goto fallback */
+ if (!next_pg)
+ return 0;
- if (next_pg >= 0) {
- if (mdb_read_pg(mdb, next_pg)) {
- table->cur_phys_pg = next_pg;
- return table->cur_phys_pg;
- } else {
+ if (!mdb_read_pg(mdb, next_pg)) {
+ fprintf(stderr, "error: reading page %d failed.\n", next_pg);
return 0;
}
+
+ table->cur_phys_pg = next_pg;
+ if (mdb->pg_buf[0]==MDB_PAGE_DATA && mdb_get_int32(mdb->pg_buf, 4)==entry->table_pg)
+ return table->cur_phys_pg;
+
+ /* On rare occasion, mdb_map_find_next will return a wrong page */
+ /* Found in a big file, over 4,000,000 records */
+ fprintf(stderr,
+ "warning: page %d from map doesn't match: Type=%d, buf[4..7]=%d Expected table_pg=%d\n",
+ next_pg, mdb_get_int32(mdb->pg_buf, 4), entry->table_pg);
}
fprintf(stderr, "Warning: defaulting to brute force read\n");
#endif
@@ -339,7 +356,7 @@
do {
if (!mdb_read_pg(mdb, table->cur_phys_pg++))
return 0;
- } while (mdb->pg_buf[0]!=0x01 || mdb_get_int32(mdb->pg_buf, 4)!=entry->table_pg);
+ } while (mdb->pg_buf[0]!=MDB_PAGE_DATA || mdb_get_int32(mdb->pg_buf, 4)!=entry->table_pg);
/* fprintf(stderr,"returning new page %ld\n", table->cur_phys_pg); */
return table->cur_phys_pg;
}
@@ -396,7 +413,7 @@
} else {
rows = mdb_get_int16(mdb->pg_buf,fmt->row_count_offset);
- /* if at end of page, find a new page */
+ /* if at end of page, find a new data page */
if (table->cur_row >= rows) {
table->cur_row=0;
Index: mdbtools-0.6pre1/include/mdbtools.h
===================================================================
--- mdbtools-0.6pre1.orig/include/mdbtools.h
+++ mdbtools-0.6pre1/include/mdbtools.h
@@ -511,7 +511,7 @@
/* map.c */
extern guint32 mdb_map_find_next_freepage(MdbTableDef *table, int row_size);
-extern guint32 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg);
+extern gint32 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg);
/* props.c */
extern GPtrArray *mdb_read_props_list(gchar *kkd, int len);
Index: mdbtools-0.6pre1/src/libmdb/map.c
===================================================================
--- mdbtools-0.6pre1.orig/src/libmdb/map.c
+++ mdbtools-0.6pre1/src/libmdb/map.c
@@ -23,7 +23,7 @@
#include "dmalloc.h"
#endif
-static guint32
+static gint32
mdb_map_find_next0(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
{
guint32 pgnum, i, usage_bitlen;
@@ -42,7 +42,7 @@
/* didn't find anything */
return 0;
}
-static int
+static gint32
mdb_map_find_next1(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
{
guint32 map_ind, max_map_pgs, offset, usage_bitlen;
@@ -83,7 +83,10 @@
/* didn't find anything */
return 0;
}
-guint32
+
+/* returns 0 on EOF */
+/* returns -1 on error (unsupported map type) */
+gint32
mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
{
if (map[0] == 0) {
@@ -119,6 +122,9 @@
/* allocate new page */
pgnum = mdb_alloc_page(table);
return pgnum;
+ } else if (pgnum==-1) {
+ fprintf(stderr, "Error: mdb_map_find_next_freepage error while reading maps.\n");
+ exit(1);
}
cur_pg = pgnum;
Index: mdbtools-0.6pre1/src/util/prfreemap.c
===================================================================
--- mdbtools-0.6pre1.orig/src/util/prfreemap.c
+++ mdbtools-0.6pre1/src/util/prfreemap.c
@@ -48,7 +48,7 @@
if (map_buf[0]==0)
while (map_buf[map_sz]==0xff) map_sz--;
- while (pgnum = mdb_map_find_next(mdb, map_buf, map_sz, pgnum)) {
+ while ((pgnum = mdb_map_find_next(mdb, map_buf, map_sz, pgnum))>0) {
printf("%6lu ",(long unsigned) pgnum);
if (coln==10) {
printf("\n");