From 4237ff3757cc248d27a7745b38a1ead06652b9c1 Mon Sep 17 00:00:00 2001 From: Arturo Espinosa Date: Thu, 18 Jun 2020 15:40:55 -0500 Subject: [PATCH 1/3] pgdbf.h: Defined valid MEMO types. Source: http://www.yaldex.com/fox_pro_tutorial/html/74f53aef-fd56-4f1a-a413-4f045922db21.htm (a Microsoft Tutorial legacy mirror) exitwitherror: Flush the standard output in case data is still buffered. Some processed output was being left out if the program crashed or exit. --- src/pgdbf.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pgdbf.h b/src/pgdbf.h index d14e4b4..315f122 100644 --- a/src/pgdbf.h +++ b/src/pgdbf.h @@ -46,6 +46,9 @@ #define NUMERICMEMOSTYLE 0 #define PACKEDMEMOSTYLE 1 +#define MEMOTYPEPICTURE 0 +#define MEMOTYPETEXT 1 + /* Don't edit this! It's defined in the XBase specification. */ #define XBASEFIELDNAMESIZE 11 @@ -244,6 +247,9 @@ static char* convertcharset(const char* inputstring, size_t* inputsize) #endif static void exitwitherror(const char *message, const int systemerror) { + /* Flush the standard output in case data is still buffered */ + fflush(stdout); + /* Print the given error message to stderr, then exit. If systemerror * is true, then use perror to explain the value in errno. */ if(systemerror) { From f5840afbfc8bae0d5e4bbb4a701cd05353895adf Mon Sep 17 00:00:00 2001 From: Arturo Espinosa Date: Thu, 18 Jun 2020 15:49:11 -0500 Subject: [PATCH 2/3] src/pgdbf.c: Added MEMO corrupted data detection and warnings. pgdbf was crashing on me due to a mismatched dbf/fpt combo. Added code to prevent calling safeprintbuf with a memo length that in combination with the memo start offset, exceeds the memo file length. Added -w option to activate warnings that allow the detection of corrupted MEMO data. Added help text for the new -w option. --- src/pgdbf.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/src/pgdbf.c b/src/pgdbf.c index aefa73b..d2ca86b 100644 --- a/src/pgdbf.c +++ b/src/pgdbf.c @@ -30,7 +30,7 @@ #include #include "pgdbf.h" -#define STANDARDOPTS "cCdDeEhm:i:nNpPqQrRtTuU" +#define STANDARDOPTS "cCdDeEhm:i:nNpPqQrRtTuUw:" int main(int argc, char **argv) { /* Describing the DBF file */ @@ -66,6 +66,9 @@ int main(int argc, char **argv) { size_t memofilesize; size_t memorecordoffset; + int32_t memolength; + int32_t memotype; + /* Processing and misc */ IGNFIELD *ignorefields; char *istr; @@ -112,6 +115,8 @@ int main(int argc, char **argv) { int optusetransaction = 1; int optusetruncatetable = 0; int opttrimpadding = 1; + int optmemowarn = 0; + uint32_t optmemowarnexcess = 0; /* Describing the PostgreSQL table */ char *tablename; @@ -220,6 +225,10 @@ int main(int argc, char **argv) { case 'U': optusetruncatetable = 0; break; + case 'w': + optmemowarn = 1; + sscanf (optarg, "%u", &optmemowarnexcess); + break; case 'h': default: /* If we got here because someone requested '-h', exit @@ -238,9 +247,9 @@ int main(int argc, char **argv) { if(optexitcode != -1) { printf( #if defined(HAVE_ICONV) - "Usage: %s [-cCdDeEhtTuU] [-s encoding] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n" + "Usage: %s [-cCdDeEhtTuU] [-w excesslength] [-s encoding] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n" #else - "Usage: %s [-cCdDeEhtTuU] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n" + "Usage: %s [-cCdDeEhtTuU] [-w excesslength] [-m memofilename] [-i fieldname1,fieldname2,fieldnameN] filename [indexcolumn ...]\n" #endif "Convert the named XBase file into PostgreSQL format\n" "\n" @@ -268,6 +277,7 @@ int main(int argc, char **argv) { " -T do not use an enclosing transaction\n" " -u issue a 'TRUNCATE' command before inserting data\n" " -U do not issue a 'TRUNCATE' command before inserting data (default)\n" + " -w warn of corrupted MEMO fields, excesslenght 0 disables check for excess length\n" "\n" #if defined(HAVE_ICONV) "If you don't specify an encoding via '-s', the data will be printed as is.\n" @@ -769,18 +779,53 @@ int main(int argc, char **argv) { s++; } } + if(memoblocknumber) { memorecordoffset = memoblocksize * memoblocknumber; if(memorecordoffset >= memofilesize) { - exitwitherror("A memo record past the end of the memofile was requested", 0); + if(optmemowarn) { + printf("CORRUPT:OFFSET::Memo record past end of memofile requested"); + fprintf(stderr, "CORRUPT:OFFSET::Memo record past end of memofile requested\n"); + break; + } + exitwitherror("A memo record past the end of the memofile was requested", 0); } + memorecord = memomap + memorecordoffset; if(memofileisdbase3) { t = strchr(memorecord, 0x1A); safeprintbuf(memorecord, t - memorecord, opttrimpadding); } else { - safeprintbuf(memorecord + 8, sbigint32_t(memorecord + 4), opttrimpadding); + memotype = sbigint32_t(memorecord); + if(optmemowarn && memotype != MEMOTYPETEXT && memotype != MEMOTYPEPICTURE) { + printf("CORRUPT:TYPE:type 0x%08xd,offset %lu:Only types 0 (PICTURE) and 1 (TEXT) allowed", memotype, memorecordoffset); + fprintf(stderr, "CORRUPT:TYPE:type 0x%08xd,offset %lu:Only types 0 (PICTURE) and 1 (TEXT) allowed\n", memotype, memorecordoffset); + break; + } + exitwitherror("A memo record with an invalid type was specified", 0); } + + memolength = sbigint32_t(memorecord + 4); + if(optmemowarn && memolength == 0) { + printf("CORRUPT:ZERO:offset %lu:Memo of length 0 specified", memorecordoffset); + fprintf(stderr, "CORRUPT:ZERO:offset %lu:Memo of length 0 specified\n", memorecordoffset); + break; + } + if(optmemowarn && optmemowarnexcess && memolength > optmemowarnexcess) { + printf("CORRUPT:EXCESS:length %d,offset %lu:Memo of excess length specified", memolength, memorecordoffset); + fprintf(stderr, "CORRUPT:EXCESS:length %d,offset %lu:Memo of excess length specified\n", memolength, memorecordoffset); + break; + } + if(memorecordoffset + 8 + memolength > memofilesize) { + if(optmemowarn) { + printf("CORRUPT:TOOLONG:length %d,offset %lu:Memo of length past the end of memofile specified", memolength, memorecordoffset); + fprintf(stderr, "CORRUPT:TOOLONG:length %d,offset %lu:Memo of length past the end of memofile specified\n", memolength, memorecordoffset); + break; + } + exitwitherror("A memo record with a length past the end of the memofile was specified", 0); + } + + safeprintbuf(memorecord + 8, memolength, opttrimpadding); } break; case 'F': From 9ac67b626fbf0855f43771189861b86087cb1295 Mon Sep 17 00:00:00 2001 From: Arturo Espinosa Date: Thu, 18 Jun 2020 15:50:53 -0500 Subject: [PATCH 3/3] doc/pgdbf.man: Added documentation for the new -w option for warnings. -w excesslength Warn of corrupted MEMO fields, by reporting errors on the output and through standard error. excesslenght specifies a MEMO length beyond which a MEMO record will be considered corrupt. A value of 0 (zero) will disable this heuristic, but the other warnings will still be issued (invalid start offset, invalid type, zero length and length beyond end of file). Conversion will not stop when warnings are issued. --- doc/pgdbf.man | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/pgdbf.man b/doc/pgdbf.man index d04d167..e50bdbb 100644 --- a/doc/pgdbf.man +++ b/doc/pgdbf.man @@ -4,7 +4,7 @@ pgdbf \- convert XBase / FoxPro tables to PostgreSQL .SH SYNOPSIS .B pgdbf -[\-cCdDeEhqQtTuU] [-m memofile] filename [indexcolumn ...] +[\-cCdDeEhqQtTuU] [-w excesslength] [-m memofile] filename [indexcolumn ...] .SH DESCRIPTION PgDBF is a program for converting XBase databases - particularly FoxPro @@ -156,6 +156,14 @@ statement to clear the contents of a table before copying data into it. Suppress the .B TRUNCATE TABLE statement. Default. +.TP +.B -w excesslength +Warn of corrupted MEMO fields, by reporting errors on the output and through +standard error. excesslenght specifies a MEMO length beyond which a MEMO record +will be considered corrupt. A value of 0 (zero) will disable this heuristic, +but the other warnings will still be issued (invalid start offset, invalid type, +zero length and length beyond end of file). Conversion will not stop when warnings +are issued. .SH "OPTION NOTES" The