pg_dump large file support

Started by Oliver Elphickover 23 years ago4 messages
#1Oliver Elphick
olly@lfix.co.uk
1 attachment(s)

I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.

This needs an additional option from autoconf, because pg_dump prints a
hex offset which needs to be either %Lx or %llx, but there is no hex
equivalent to INT64_FORMAT. To get this working I have arbitrarily
defined OFF_T_FORMATX as "%lld", but that needs to be changed before the
patch is incorporated.

Peter E suggested that there was no need for the macro definitions in
pg_backup.h but I wasn't clear how else to achieve what was needed.

--
Oliver Elphick Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839 932A 614D 4C34 3E1D 0C1C
========================================
"Praying always with all prayer and supplication in the
Spirit, and watching thereunto with all perseverance
and supplication for all saints." Ephesians 6:18

Attachments:

pg_dump.patchtext/x-patch; charset=ISO-8859-15; name=pg_dump.patchDownload
Index: src/bin/pg_dump/Makefile
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/Makefile,v
retrieving revision 1.36
diff -u -r1.36 Makefile
--- src/bin/pg_dump/Makefile	2002/07/27 20:10:05	1.36
+++ src/bin/pg_dump/Makefile	2002/08/15 10:31:53
@@ -18,6 +18,8 @@
 
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 
+CFLAGS +=  -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+
 all: submake-libpq submake-libpgport pg_dump pg_restore pg_dumpall
 
 pg_dump: pg_dump.o common.o $(OBJS) $(libpq_builddir)/libpq.a 
Index: src/bin/pg_dump/pg_backup.h
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup.h,v
retrieving revision 1.20
diff -u -r1.20 pg_backup.h
--- src/bin/pg_dump/pg_backup.h	2002/07/04 15:35:07	1.20
+++ src/bin/pg_dump/pg_backup.h	2002/08/15 10:31:53
@@ -27,6 +27,25 @@
 
 #include "libpq-fe.h"
 
+/*
+ * Large file support
+ */
+
+#ifdef _LARGEFILE_SOURCE
+ #define FSEEK fseeko
+ #define FTELL ftello
+ #define OFF_T_FORMAT INT64_FORMAT
+ #define OFF_T_FORMATX "%llx"            /* needs an autoconf setting */
+ typedef off_t OFF_T;
+#else
+ #define FSEEK fseek
+ #define FTELL ftell
+ #define OFF_T_FORMAT "%ld"
+ #define OFF_T_FORMATX "%lx"
+ typedef long int OFF_T;
+#endif
+
+
 #define atooid(x)  ((Oid) strtoul((x), NULL, 10))
 #define oidcmp(x,y) ( ((x) < (y) ? -1 : ((x) > (y)) ?  1 : 0) )
 #define oideq(x,y) ( (x) == (y) )
Index: src/bin/pg_dump/pg_backup_archiver.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_archiver.c,v
retrieving revision 1.53
diff -u -r1.53 pg_backup_archiver.c
--- src/bin/pg_dump/pg_backup_archiver.c	2002/08/10 16:57:31	1.53
+++ src/bin/pg_dump/pg_backup_archiver.c	2002/08/15 10:31:55
@@ -1561,7 +1561,7 @@
 	}
 
 	/* If we can't seek, then mark the header as read */
-	if (fseek(fh, 0, SEEK_SET) != 0)
+	if (FSEEK(fh, 0, SEEK_SET) != 0)
 	{
 		/*
 		 * NOTE: Formats that use the looahead buffer can unset this in
Index: src/bin/pg_dump/pg_backup_custom.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_custom.c,v
retrieving revision 1.19
diff -u -r1.19 pg_backup_custom.c
--- src/bin/pg_dump/pg_backup_custom.c	2002/05/29 01:38:56	1.19
+++ src/bin/pg_dump/pg_backup_custom.c	2002/08/15 10:31:56
@@ -81,13 +81,13 @@
 	char	   *zlibIn;
 	int			inSize;
 	int			hasSeek;
-	int			filePos;
+	OFF_T			filePos;
 	int			dataStart;
 } lclContext;
 
 typedef struct
 {
-	int			dataPos;
+	OFF_T			dataPos;
 	int			dataLen;
 } lclTocEntry;
 
@@ -99,7 +99,7 @@
 static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
 static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
 static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
-static int	_getFilePos(ArchiveHandle *AH, lclContext *ctx);
+static OFF_T	_getFilePos(ArchiveHandle *AH, lclContext *ctx);
 static int	_DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);
 
 static char *modulename = gettext_noop("custom archiver");
@@ -188,7 +188,7 @@
 		if (!AH->FH)
 			die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));
 
-		ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+		ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
 
 	}
 	else
@@ -201,7 +201,7 @@
 		if (!AH->FH)
 			die_horribly(AH, modulename, "could not open archive file %s: %s\n", AH->fSpec, strerror(errno));
 
-		ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+		ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
 
 		ReadHead(AH);
 		ReadToc(AH);
@@ -285,7 +285,7 @@
 {
 	lclTocEntry *ctx = (lclTocEntry *) te->formatData;
 
-	ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen);
+	ahprintf(AH, "-- Data Pos: " INT64_FORMAT " (Length %d)\n", ctx->dataPos, ctx->dataLen);
 }
 
 /*
@@ -486,7 +486,7 @@
 
 		/* Grab it */
 
-		if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0)
+		if (FSEEK(AH->FH, tctx->dataPos, SEEK_SET) != 0)
 			die_horribly(AH, modulename, "error during file seek: %s\n", strerror(errno));
 
 		_readBlockHeader(AH, &blkType, &id);
@@ -816,12 +816,12 @@
 _CloseArchive(ArchiveHandle *AH)
 {
 	lclContext *ctx = (lclContext *) AH->formatData;
-	int			tpos;
+	OFF_T			tpos;
 
 	if (AH->mode == archModeWrite)
 	{
 		WriteHead(AH);
-		tpos = ftell(AH->FH);
+		tpos = FTELL(AH->FH);
 		WriteToc(AH);
 		ctx->dataStart = _getFilePos(AH, ctx);
 		WriteDataChunks(AH);
@@ -834,7 +834,7 @@
 		 */
 		if (ctx->hasSeek)
 		{
-			fseek(AH->FH, tpos, SEEK_SET);
+			FSEEK(AH->FH, tpos, SEEK_SET);
 			WriteToc(AH);
 		}
 	}
@@ -853,14 +853,14 @@
 /*
  * Get the current position in the archive file.
  */
-static int
+static OFF_T
 _getFilePos(ArchiveHandle *AH, lclContext *ctx)
 {
-	int			pos;
+	OFF_T			pos;
 
 	if (ctx->hasSeek)
 	{
-		pos = ftell(AH->FH);
+		pos = FTELL(AH->FH);
 		if (pos != ctx->filePos)
 		{
 			write_msg(modulename, "WARNING: ftell mismatch with expected position -- ftell ignored\n");
Index: src/bin/pg_dump/pg_backup_files.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_files.c,v
retrieving revision 1.16
diff -u -r1.16 pg_backup_files.c
--- src/bin/pg_dump/pg_backup_files.c	2002/05/29 01:38:56	1.16
+++ src/bin/pg_dump/pg_backup_files.c	2002/08/15 10:31:56
@@ -60,7 +60,7 @@
 typedef struct
 {
 	int			hasSeek;
-	int			filePos;
+	OFF_T			filePos;
 	FILE	   *blobToc;
 } lclContext;
 
@@ -137,7 +137,7 @@
 		if (AH->FH == NULL)
 			die_horribly(NULL, modulename, "could not open output file: %s\n", strerror(errno));
 
-		ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+		ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
 
 		if (AH->compression < 0 || AH->compression > 9)
 			AH->compression = Z_DEFAULT_COMPRESSION;
@@ -155,7 +155,7 @@
 		if (AH->FH == NULL)
 			die_horribly(NULL, modulename, "could not open input file: %s\n", strerror(errno));
 
-		ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);
+		ctx->hasSeek = (FSEEK(AH->FH, 0, SEEK_CUR) == 0);
 
 		ReadHead(AH);
 		ReadToc(AH);
Index: src/bin/pg_dump/pg_backup_tar.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/pg_backup_tar.c,v
retrieving revision 1.24
diff -u -r1.24 pg_backup_tar.c
--- src/bin/pg_dump/pg_backup_tar.c	2002/07/04 15:35:07	1.24
+++ src/bin/pg_dump/pg_backup_tar.c	2002/08/15 10:31:57
@@ -187,7 +187,7 @@
 		 */
 		/* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
 
-		ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+		ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);
 
 		if (AH->compression < 0 || AH->compression > 9)
 			AH->compression = Z_DEFAULT_COMPRESSION;
@@ -224,7 +224,7 @@
 
 		ctx->tarFHpos = 0;
 
-		ctx->hasSeek = (fseek(ctx->tarFH, 0, SEEK_CUR) == 0);
+		ctx->hasSeek = (FSEEK(ctx->tarFH, 0, SEEK_CUR) == 0);
 
 		/*
 		 * Forcibly unmark the header as read since we use the lookahead
@@ -1011,9 +1011,9 @@
 	/*
 	 * Find file len & go back to start.
 	 */
-	fseek(tmp, 0, SEEK_END);
-	th->fileLen = ftell(tmp);
-	fseek(tmp, 0, SEEK_SET);
+	FSEEK(tmp, 0, SEEK_END);
+	th->fileLen = FTELL(tmp);
+	FSEEK(tmp, 0, SEEK_SET);
 
 	_tarWriteHeader(th);
 
@@ -1130,10 +1130,10 @@
 	while (!gotBlock)
 	{
 #if 0
-		if (ftell(ctx->tarFH) != ctx->tarFHpos)
+		if (FTELL(ctx->tarFH) != ctx->tarFHpos)
 			die_horribly(AH, modulename,
 						 "mismatch in actual vs. predicted file position (%d vs. %d)\n",
-						 ftell(ctx->tarFH), ctx->tarFHpos);
+						 FTELL(ctx->tarFH), ctx->tarFHpos);
 #endif
 
 		/* Save the pos for reporting purposes */
@@ -1179,8 +1179,9 @@
 	if (chk != sum)
 		die_horribly(AH, modulename,
 					 "corrupt tar header found in %s "
-		"(expected %d (%o), computed %d (%o)) file position %ld (%lx)\n",
-					 &tag[0], sum, sum, chk, chk, ftell(ctx->tarFH), ftell(ctx->tarFH));
+		"(expected %d (%o), computed %d (%o)) file position "
+		OFF_T_FORMAT " (" OFF_T_FORMATX ")\n",
+					 &tag[0], sum, sum, chk, chk, FTELL(ctx->tarFH), FTELL(ctx->tarFH));
 
 	th->targetFile = strdup(tag);
 	th->fileLen = len;
#2Oliver Elphick
olly@lfix.co.uk
In reply to: Oliver Elphick (#1)
Re: pg_dump large file support

On Thu, 2002-08-15 at 11:41, Oliver Elphick wrote:

I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.

Index: src/bin/pg_dump/Makefile
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/pg_dump/Makefile,v
retrieving revision 1.36
diff -u -r1.36 Makefile
--- src/bin/pg_dump/Makefile	2002/07/27 20:10:05	1.36
+++ src/bin/pg_dump/Makefile	2002/08/15 10:31:53
@@ -18,6 +18,8 @@

override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)

+CFLAGS +=  -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
+

I just realised this is the wrong place to put this definition, since it
will act unconditonally and attempt to compile large file support even
where it is not available. This too ought to come from autoconf, I
think.

--
Oliver Elphick Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839 932A 614D 4C34 3E1D 0C1C
========================================
"Praying always with all prayer and supplication in the
Spirit, and watching thereunto with all perseverance
and supplication for all saints." Ephesians 6:18

#3Peter Eisentraut
peter_e@gmx.net
In reply to: Oliver Elphick (#1)
Re: pg_dump large file support

Oliver Elphick writes:

I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.

Hmm, not quite right, but since you've indicated that you're not familiar
with Autoconf I can take it from here.

--
Peter Eisentraut peter_e@gmx.net

#4Oliver Elphick
olly@lfix.co.uk
In reply to: Peter Eisentraut (#3)
Re: pg_dump large file support

On Thu, 2002-08-15 at 20:30, Peter Eisentraut wrote:

Oliver Elphick writes:

I've implemented large file support for pg_dump, in what I hope is a
portable fashion. Please review the attached patch.

Hmm, not quite right, but since you've indicated that you're not familiar
with Autoconf I can take it from here.

Thanks, Peter
--
Oliver Elphick Oliver.Elphick@lfix.co.uk
Isle of Wight, UK
http://www.lfix.co.uk/oliver
GPG: 1024D/3E1D0C1C: CA12 09E0 E8D5 8870 5839 932A 614D 4C34 3E1D 0C1C
========================================
"Praying always with all prayer and supplication in the
Spirit, and watching thereunto with all perseverance
and supplication for all saints." Ephesians 6:18