pg_dump large file support
I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.
This needs an additional option from autoconf, because pg_dump prints a
hex offset which needs to be either %Lx or %llx, but there is no hex
equivalent to INT64_FORMAT. To get this working I have arbitrarily
defined OFF_T_FORMATX as "%lld", but that needs to be changed before the
patch is incorporated.
Peter E suggested that there was no need for the macro definitions in
pg_backup.h but I wasn't clear how else to achieve what was needed.
--
Oliver Elphick Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839 932A 614D 4C34 3E1D 0C1C
========================================
"Praying always with all prayer and supplication in the
Spirit, and watching thereunto with all perseverance
and supplication for all saints." Ephesians 6:18
Attachments:
pg_dump.patchtext/x-patch; charset=ISO-8859-15; name=pg_dump.patchDownload
Index: src/bin/pg_dump/Makefile
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/Makefile,v
retrieving revision 1.36
diff -u -r1.36 Makefile
--- src/bin/pg_dump/Makefile 2002/07/27 20:10:05 1.36
+++ src/bin/pg_dump/Makefile 2002/08/15 10:31:53
@@ -18,6 +18,8 @@
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
+CFLAGS += -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+
all: submake-libpq submake-libpgport pg_dump pg_restore pg_dumpall
pg_dump: pg_dump.o common.o $(OBJS) $(libpq_builddir)/libpq.a
Index: src/bin/pg_dump/pg_backup.h
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup.h,v
retrieving revision 1.20
diff -u -r1.20 pg_backup.h
--- src/bin/pg_dump/pg_backup.h 2002/07/04 15:35:07 1.20
+++ src/bin/pg_dump/pg_backup.h 2002/08/15 10:31:53
@@ -27,6 +27,25 @@
#include "libpq-fe.h"
+/*
+ * Large file support
+ */
+
+#ifdef _LARGEFILE_SOURCE
+ #define FSEEK fseeko
+ #define FTELL ftello
+ #define OFF_T_FORMAT INT64_FORMAT
+ #define OFF_T_FORMATX "%llx" /* needs an autoconf setting */
+ typedef off_t OFF_T;
+#else
+ #define FSEEK fseek
+ #define FTELL ftell
+ #define OFF_T_FORMAT "%ld"
+ #define OFF_T_FORMATX "%lx"
+ typedef long int OFF_T;
+#endif
+
+
#define atooid(x) ((Oid) strtoul((x), NULL, 10))
#define oidcmp(x,y) ( ((x) < (y) ? -1 : ((x) > (y)) ? 1 : 0) )
#define oideq(x,y) ( (x) == (y) )
Index: src/bin/pg_dump/pg_backup_archiver.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_archiver.c,v
retrieving revision 1.53
diff -u -r1.53 pg_backup_archiver.c
--- src/bin/pg_dump/pg_backup_archiver.c 2002/08/10 16:57:31 1.53
+++ src/bin/pg_dump/pg_backup_archiver.c 2002/08/15 10:31:55
@@ -1561,7 +1561,7 @@
}
/* If we can't seek, then mark the header as read */
- if (fseek(fh, 0, SEEK_SET) != 0)
+ if (FSEEK(fh, 0, SEEK_SET) != 0)
{
/*
* NOTE: Formats that use the looahead buffer can unset this in
Index: src/bin/pg_dump/pg_backup_custom.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_custom.c,v
retrieving revision 1.19
diff -u -r1.19 pg_backup_custom.c
--- src/bin/pg_dump/pg_backup_custom.c 2002/05/29 01:38:56 1.19
+++ src/bin/pg_dump/pg_backup_custom.c 2002/08/15 10:31:56
@@ -81,13 +81,13 @@
char *zlibIn;
int inSize;
int hasSeek;
- int filePos;
+ OFF_T filePos;
int dataStart;
} lclContext;
typedef struct
{
- int dataPos;
+ OFF_T dataPos;
int dataLen;
} lclTocEntry;
@@ -99,7 +99,7 @@
static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
-static int _getFilePos(ArchiveHandle *AH, lclContext *ctx);
+static OFF_T _getFilePos(ArchiveHandle *AH, lclContext *ctx);
static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);
static char *modulename = gettext_noop("custom archiver");
@@ -188,7 +188,7 @@
if (!AH->FH)
die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
}
else
@@ -201,7 +201,7 @@
if (!AH->FH)
die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
ReadHead(AH);
ReadToc(AH);
@@ -285,7 +285,7 @@
{
lclTocEntry *ctx = (lclTocEntry *) te->formatData;
- ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen);
+ ahprintf(AH, "-- Data Pos: " INT64_FORMAT " (Length %d)\n", ctx->dataPos, ctx->dataLen);
}
/*
@@ -486,7 +486,7 @@
/* Grab it */
- if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0)
+ if (FSEEK(AH->FH, tctx->dataPos, SEEK_SET) != 0)
die_horribly(AH, modulename, "error during file seek: %s\n", strerror(errno));
_readBlockHeader(AH, &blkType, &id);
@@ -816,12 +816,12 @@
_CloseArchive(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
- int tpos;
+ OFF_T tpos;
if (AH->mode == archModeWrite)
{
WriteHead(AH);
- tpos = ftell(AH->FH);
+ tpos = FTELL(AH->FH);
WriteToc(AH);
ctx->dataStart = _getFilePos(AH, ctx);
WriteDataChunks(AH);
@@ -834,7 +834,7 @@
*/
if (ctx->hasSeek)
{
- fseek(AH->FH, tpos, SEEK_SET);
+ FSEEK(AH->FH, tpos, SEEK_SET);
WriteToc(AH);
}
}
@@ -853,14 +853,14 @@
/*
* Get the current position in the archive file.
*/
-static int
+static OFF_T
_getFilePos(ArchiveHandle *AH, lclContext *ctx)
{
- int pos;
+ OFF_T pos;
if (ctx->hasSeek)
{
- pos = ftell(AH->FH);
+ pos = FTELL(AH->FH);
if (pos != ctx->filePos)
{
write_msg(modulename, "WARNING: ftell mismatch with expected position -- ftell ignored\n");
Index: src/bin/pg_dump/pg_backup_files.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_files.c,v
retrieving revision 1.16
diff -u -r1.16 pg_backup_files.c
--- src/bin/pg_dump/pg_backup_files.c 2002/05/29 01:38:56 1.16
+++ src/bin/pg_dump/pg_backup_files.c 2002/08/15 10:31:56
@@ -60,7 +60,7 @@
typedef struct
{
int hasSeek;
- int filePos;
+ OFF_T filePos;
FILE *blobToc;
} lclContext;
@@ -137,7 +137,7 @@
if (AH->FH == NULL)
die_horribly(NULL, modulename, "could not open output file: %s\n", strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
if (AH->compression < 0 || AH->compression > 9)
AH->compression = Z_DEFAULT_COMPRESSION;
@@ -155,7 +155,7 @@
if (AH->FH == NULL)
die_horribly(NULL, modulename, "could not open input file: %s\n", strerror(errno));
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
ReadHead(AH);
ReadToc(AH);
Index: src/bin/pg_dump/pg_backup_tar.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_tar.c,v
retrieving revision 1.24
diff -u -r1.24 pg_backup_tar.c
--- src/bin/pg_dump/pg_backup_tar.c 2002/07/04 15:35:07 1.24
+++ src/bin/pg_dump/pg_backup_tar.c 2002/08/15 10:31:57
@@ -187,7 +187,7 @@
*/
/* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
- ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);
if (AH->compression < 0 || AH->compression > 9)
AH->compression = Z_DEFAULT_COMPRESSION;
@@ -224,7 +224,7 @@
ctx->tarFHpos = 0;
- ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+ ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);
/*
* Forcibly unmark the header as read since we use the lookahead
@@ -1011,9 +1011,9 @@
/*
* Find file len & go back to start.
*/
- fseek(tmp, 0, SEEK_END);
- th->fileLen = ftell(tmp);
- fseek(tmp, 0, SEEK_SET);
+ FSEEK(tmp, 0, SEEK_END);
+ th->fileLen = FTELL(tmp);
+ FSEEK(tmp, 0, SEEK_SET);
_tarWriteHeader(th);
@@ -1130,10 +1130,10 @@
while (!gotBlock)
{
#if 0
- if (ftell(ctx->tarFH) != ctx->tarFHpos)
+ if (FTELL(ctx->tarFH) != ctx->tarFHpos)
die_horribly(AH, modulename,
"mismatch in actual vs. predicted file position (%d vs. %d)\n",
- ftell(ctx->tarFH), ctx->tarFHpos);
+ FTELL(ctx->tarFH), ctx->tarFHpos);
#endif
/* Save the pos for reporting purposes */
@@ -1179,8 +1179,9 @@
if (chk != sum)
die_horribly(AH, modulename,
"corrupt tar header found in %s "
- "(expected %d (%o), computed %d (%o)) file position %ld (%lx)\n",
- &tag[0], sum, sum, chk, chk, ftell(ctx->tarFH), ftell(ctx->tarFH));
+ "(expected %d (%o), computed %d (%o)) file position "
+ OFF_T_FORMAT " (" OFF_T_FORMATX ")\n",
+ &tag[0], sum, sum, chk, chk, FTELL(ctx->tarFH), FTELL(ctx->tarFH));
th->targetFile = strdup(tag);
th->fileLen = len;
On Thu, 2002-08-15 at 11:41, Oliver Elphick wrote:
I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.
Index: src/bin/pg_dump/Makefile =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/Makefile,v retrieving revision 1.36 diff -u -r1.36 Makefile --- src/bin/pg_dump/Makefile 2002/07/27 20:10:05 1.36 +++ src/bin/pg_dump/Makefile 2002/08/15 10:31:53 @@ -18,6 +18,8 @@override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
+CFLAGS += -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 +
I just realised this is the wrong place to put this definition, since it
will act unconditonally and attempt to compile large file support even
where it is not available. This too ought to come from autoconf, I
think.
--
Oliver Elphick Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839 932A 614D 4C34 3E1D 0C1C
========================================
"Praying always with all prayer and supplication in the
Spirit, and watching thereunto with all perseverance
and supplication for all saints." Ephesians 6:18
Oliver Elphick writes:
I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.
Hmm, not quite right, but since you've indicated that you're not familiar
with Autoconf I can take it from here.
--
Peter Eisentraut peter_e@gmx.net
On Thu, 2002-08-15 at 20:30, Peter Eisentraut wrote:
Oliver Elphick writes:
I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.Hmm, not quite right, but since you've indicated that you're not familiar
with Autoconf I can take it from here.
Thanks, Peter
--
Oliver Elphick Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839 932A 614D 4C34 3E1D 0C1C
========================================
"Praying always with all prayer and supplication in the
Spirit, and watching thereunto with all perseverance
and supplication for all saints." Ephesians 6:18