diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index df84054..49cb7ac 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -178,6 +178,7 @@ Complete list of usable sgml source files in this directory. + diff --git a/doc/src/sgml/ref/pg_xlogdump.sgml b/doc/src/sgml/ref/pg_xlogdump.sgml new file mode 100644 index 0000000..7a27c7b --- /dev/null +++ b/doc/src/sgml/ref/pg_xlogdump.sgml @@ -0,0 +1,76 @@ + + + + + pg_xlogdump + 1 + Application + + + + pg_xlogdump + Display the write-ahead log of a PostgreSQL database cluster + + + + pg_xlogdump + + + + + pg_xlogdump + + xlogrecptr + filename + + directory + xlogrecptr + timelineid + + + + + + Description + + pg_xlogdump display the write-ahead log (WAL) and is only + useful for debugging or educational purposes. + + + + This utility can only be run by the user who installed the server, because + it requires read access to the data directory. It does not perform any + modifications. + + + + + Options + + + The following command-line options control the location and format of the + output. + + + + + + + Directory to find xlog files in. + + + + + + + + + Notes + + Can give wrong results when the server is running. + + + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index 0872168..fed1fdd 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -225,6 +225,7 @@ &pgDumpall; &pgReceivexlog; &pgRestore; + &pgXlogdump; &psqlRef; &reindexdb; &vacuumdb; diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile index 700cfd8..eb6cfc5 100644 --- a/src/backend/access/transam/Makefile +++ b/src/backend/access/transam/Makefile @@ -14,7 +14,7 @@ include $(top_builddir)/src/Makefile.global OBJS = clog.o transam.o varsup.o xact.o rmgr.o slru.o subtrans.o multixact.o \ timeline.o twophase.o twophase_rmgr.o xlog.o xlogarchive.o xlogfuncs.o \ - xlogutils.o + xlogreader.o xlogutils.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 2618c8d..4c36468 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -30,6 +30,7 @@ #include "access/twophase.h" #include "access/xact.h" #include "access/xlog_internal.h" +#include "access/xlogreader.h" #include "access/xlogutils.h" #include "catalog/catversion.h" #include "catalog/pg_control.h" @@ -531,29 +532,33 @@ static XLogSegNo openLogSegNo = 0; static uint32 openLogOff = 0; /* - * These variables are used similarly to the ones above, but for reading + * Status data for XLogPageRead. + * + * The first three are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which * will be just past that page. readLen indicates how much of the current * page has been read into readBuf, and readSource indicates where we got * the currently open file from. + * + * currentSource keeps track of which source we're currently reading from. This + * is different from readSource in that this is always set, even when we don't + * currently have a WAL file open. If lastSourceFailed is set, our last attempt + * to read from currentSource failed, and we should try another source next. */ -static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static bool readFileHeaderValidated = false; -static XLogSource readSource = 0; /* XLOG_FROM_* code */ - -/* - * Keeps track of which source we're currently reading from. This is - * different from readSource in that this is always set, even when we don't - * currently have a WAL file open. If lastSourceFailed is set, our last - * attempt to read from currentSource failed, and we should try another source - * next. - */ -static XLogSource currentSource = 0; /* XLOG_FROM_* code */ -static bool lastSourceFailed = false; +typedef struct XLogPageReadPrivate +{ + int emode; + + int readFile; + XLogSegNo readSegNo; + uint32 readOff; + uint32 readLen; + bool fetching_ckpt; /* are we fetching a checkpoint record? */ + XLogSource readSource; /* XLOG_FROM_* code */ + XLogSource currentSource; /* XLOG_FROM_* code */ + bool lastSourceFailed; +} XLogPageReadPrivate; /* * These variables track when we last obtained some WAL data to process, @@ -566,18 +571,9 @@ static bool lastSourceFailed = false; static TimestampTz XLogReceiptTime = 0; static XLogSource XLogReceiptSource = 0; /* XLOG_FROM_* code */ -/* Buffer for currently read page (XLOG_BLCKSZ bytes) */ -static char *readBuf = NULL; - -/* Buffer for current ReadRecord result (expandable) */ -static char *readRecordBuf = NULL; -static uint32 readRecordBufSize = 0; - /* State information for XLOG reading */ static XLogRecPtr ReadRecPtr; /* start of last record read */ static XLogRecPtr EndRecPtr; /* end+1 of last record read */ -static TimeLineID lastPageTLI = 0; -static TimeLineID lastSegmentTLI = 0; static XLogRecPtr minRecoveryPoint; /* local copy of * ControlFile->minRecoveryPoint */ @@ -598,7 +594,8 @@ static bool bgwriterLaunched = false; static void readRecoveryCommandFile(void); -static void exitArchiveRecovery(TimeLineID endTLI, XLogSegNo endLogSegNo); +static void exitArchiveRecovery(XLogPageReadPrivate *private, TimeLineID endTLI, + XLogSegNo endLogSegNo); static bool recoveryStopsHere(XLogRecord *record, bool *includeThis); static void recoveryPausesHere(void); static void SetLatestXTime(TimestampTz xtime); @@ -617,14 +614,15 @@ static void XLogWrite(XLogwrtRqst WriteRqst, bool flexible, bool xlog_switch); static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, bool find_free, int *max_advance, bool use_lock); -static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, - int source, bool notexistOk); -static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source); -static bool XLogPageRead(XLogRecPtr *RecPtr, int emode, bool fetching_ckpt, - bool randAccess); -static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt); -static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); +static int XLogFileRead(XLogPageReadPrivate *private, XLogSegNo segno, + int emode, TimeLineID tli, int source, bool notexistOk); +static int XLogFileReadAnyTLI(XLogPageReadPrivate *private, XLogSegNo segno, + int emode, int source); +static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, + int reqLen, char *readBuf, TimeLineID *readTLI); +static bool WaitForWALToBecomeAvailable(XLogPageReadPrivate *private, + XLogRecPtr RecPtr); +static int emode_for_corrupt_record(XLogReaderState *state, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); static void RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr endptr); @@ -632,12 +630,11 @@ static void UpdateLastRemovedPtr(char *filename); static void ValidateXLOGDirectoryStructure(void); static void CleanupBackupHistory(void); static void UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force); -static XLogRecord *ReadRecord(XLogRecPtr *RecPtr, int emode, bool fetching_ckpt); +static XLogRecord *ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, + int emode, bool fetching_ckpt); static void CheckRecoveryConsistency(void); -static bool ValidXLogPageHeader(XLogPageHeader hdr, int emode, bool segmentonly); -static bool ValidXLogRecordHeader(XLogRecPtr *RecPtr, XLogRecord *record, - int emode, bool randAccess); -static XLogRecord *ReadCheckpointRecord(XLogRecPtr RecPtr, int whichChkpt); +static XLogRecord *ReadCheckpointRecord(XLogReaderState *xlogreader, + XLogRecPtr RecPtr, int whichChkpt); static bool rescanLatestTimeLine(void); static void WriteControlFile(void); static void ReadControlFile(void); @@ -2577,8 +2574,8 @@ XLogFileOpen(XLogSegNo segno) * Otherwise, it's assumed to be already available in pg_xlog. */ static int -XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, - int source, bool notfoundOk) +XLogFileRead(XLogPageReadPrivate *private, XLogSegNo segno, int emode, + TimeLineID tli, int source, bool notfoundOk) { char xlogfname[MAXFNAMELEN]; char activitymsg[MAXFNAMELEN + 16]; @@ -2695,15 +2692,12 @@ XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, set_ps_display(activitymsg, false); /* Track source of data in assorted state variables */ - readSource = source; + private->readSource = source; XLogReceiptSource = source; /* In FROM_STREAM case, caller tracks receipt time, not me */ if (source != XLOG_FROM_STREAM) XLogReceiptTime = GetCurrentTimestamp(); - /* The file header needs to be validated on first access */ - readFileHeaderValidated = false; - return fd; } if (errno != ENOENT || !notfoundOk) /* unexpected failure? */ @@ -2719,7 +2713,8 @@ XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, * This version searches for the segment with any TLI listed in expectedTLEs. */ static int -XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source) +XLogFileReadAnyTLI(XLogPageReadPrivate *private, XLogSegNo segno, + int emode, int source) { char path[MAXPGPATH]; ListCell *cell; @@ -2744,7 +2739,8 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source) if (source == XLOG_FROM_ANY || source == XLOG_FROM_ARCHIVE) { - fd = XLogFileRead(segno, emode, tli, XLOG_FROM_ARCHIVE, true); + fd = XLogFileRead(private, segno, emode, tli, + XLOG_FROM_ARCHIVE, true); if (fd != -1) { elog(DEBUG1, "got WAL segment from archive"); @@ -2754,7 +2750,8 @@ XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source) if (source == XLOG_FROM_ANY || source == XLOG_FROM_PG_XLOG) { - fd = XLogFileRead(segno, emode, tli, XLOG_FROM_PG_XLOG, true); + fd = XLogFileRead(private, segno, emode, tli, + XLOG_FROM_PG_XLOG, true); if (fd != -1) return fd; } @@ -3187,102 +3184,6 @@ RestoreBackupBlock(XLogRecPtr lsn, XLogRecord *record, int block_index, } /* - * CRC-check an XLOG record. We do not believe the contents of an XLOG - * record (other than to the minimal extent of computing the amount of - * data to read in) until we've checked the CRCs. - * - * We assume all of the record (that is, xl_tot_len bytes) has been read - * into memory at *record. Also, ValidXLogRecordHeader() has accepted the - * record's header, which means in particular that xl_tot_len is at least - * SizeOfXlogRecord, so it is safe to fetch xl_len. - */ -static bool -RecordIsValid(XLogRecord *record, XLogRecPtr recptr, int emode) -{ - pg_crc32 crc; - int i; - uint32 len = record->xl_len; - BkpBlock bkpb; - char *blk; - size_t remaining = record->xl_tot_len; - - /* First the rmgr data */ - if (remaining < SizeOfXLogRecord + len) - { - /* ValidXLogRecordHeader() should've caught this already... */ - ereport(emode_for_corrupt_record(emode, recptr), - (errmsg("invalid record length at %X/%X", - (uint32) (recptr >> 32), (uint32) recptr))); - return false; - } - remaining -= SizeOfXLogRecord + len; - INIT_CRC32(crc); - COMP_CRC32(crc, XLogRecGetData(record), len); - - /* Add in the backup blocks, if any */ - blk = (char *) XLogRecGetData(record) + len; - for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++) - { - uint32 blen; - - if (!(record->xl_info & XLR_BKP_BLOCK(i))) - continue; - - if (remaining < sizeof(BkpBlock)) - { - ereport(emode_for_corrupt_record(emode, recptr), - (errmsg("invalid backup block size in record at %X/%X", - (uint32) (recptr >> 32), (uint32) recptr))); - return false; - } - memcpy(&bkpb, blk, sizeof(BkpBlock)); - - if (bkpb.hole_offset + bkpb.hole_length > BLCKSZ) - { - ereport(emode_for_corrupt_record(emode, recptr), - (errmsg("incorrect hole size in record at %X/%X", - (uint32) (recptr >> 32), (uint32) recptr))); - return false; - } - blen = sizeof(BkpBlock) + BLCKSZ - bkpb.hole_length; - - if (remaining < blen) - { - ereport(emode_for_corrupt_record(emode, recptr), - (errmsg("invalid backup block size in record at %X/%X", - (uint32) (recptr >> 32), (uint32) recptr))); - return false; - } - remaining -= blen; - COMP_CRC32(crc, blk, blen); - blk += blen; - } - - /* Check that xl_tot_len agrees with our calculation */ - if (remaining != 0) - { - ereport(emode_for_corrupt_record(emode, recptr), - (errmsg("incorrect total length in record at %X/%X", - (uint32) (recptr >> 32), (uint32) recptr))); - return false; - } - - /* Finally include the record header */ - COMP_CRC32(crc, (char *) record, offsetof(XLogRecord, xl_crc)); - FIN_CRC32(crc); - - if (!EQ_CRC32(record->xl_crc, crc)) - { - ereport(emode_for_corrupt_record(emode, recptr), - (errmsg("incorrect resource manager data checksum in record at %X/%X", - (uint32) (recptr >> 32), (uint32) recptr))); - return false; - } - - return true; -} - -/* * Attempt to read an XLOG record. * * If RecPtr is not NULL, try to read a record at that position. Otherwise @@ -3295,511 +3196,41 @@ RecordIsValid(XLogRecord *record, XLogRecPtr recptr, int emode) * the returned record pointer always points there. */ static XLogRecord * -ReadRecord(XLogRecPtr *RecPtr, int emode, bool fetching_ckpt) +ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode, + bool fetching_ckpt) { XLogRecord *record; - XLogRecPtr tmpRecPtr = EndRecPtr; - bool randAccess = false; - uint32 len, - total_len; - uint32 targetRecOff; - uint32 pageHeaderSize; - bool gotheader; - - if (readBuf == NULL) - { - /* - * First time through, permanently allocate readBuf. We do it this - * way, rather than just making a static array, for two reasons: (1) - * no need to waste the storage in most instantiations of the backend; - * (2) a static char array isn't guaranteed to have any particular - * alignment, whereas malloc() will provide MAXALIGN'd storage. - */ - readBuf = (char *) malloc(XLOG_BLCKSZ); - Assert(readBuf != NULL); - } - - if (RecPtr == NULL) - { - RecPtr = &tmpRecPtr; - - /* - * RecPtr is pointing to end+1 of the previous WAL record. If - * we're at a page boundary, no more records can fit on the current - * page. We must skip over the page header, but we can't do that - * until we've read in the page, since the header size is variable. - */ - } - else - { - /* - * In this case, the passed-in record pointer should already be - * pointing to a valid record starting position. - */ - if (!XRecOffIsValid(*RecPtr)) - ereport(PANIC, - (errmsg("invalid record offset at %X/%X", - (uint32) (*RecPtr >> 32), (uint32) *RecPtr))); + XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data; - /* - * Since we are going to a random position in WAL, forget any prior - * state about what timeline we were in, and allow it to be any - * timeline in expectedTLEs. We also set a flag to allow curFileTLI - * to go backwards (but we can't reset that variable right here, since - * we might not change files at all). - */ - /* see comment in ValidXLogPageHeader */ - lastPageTLI = lastSegmentTLI = 0; - randAccess = true; /* allow curFileTLI to go backwards too */ - } + /* Pass parameters to XLogPageRead */ + private->fetching_ckpt = fetching_ckpt; + private->emode = emode; /* This is the first try to read this page. */ - lastSourceFailed = false; -retry: - /* Read the page containing the record */ - if (!XLogPageRead(RecPtr, emode, fetching_ckpt, randAccess)) - return NULL; - - pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) readBuf); - targetRecOff = (*RecPtr) % XLOG_BLCKSZ; - if (targetRecOff == 0) - { - /* - * At page start, so skip over page header. The Assert checks that - * we're not scribbling on caller's record pointer; it's OK because we - * can only get here in the continuing-from-prev-record case, since - * XRecOffIsValid rejected the zero-page-offset case otherwise. - */ - Assert(RecPtr == &tmpRecPtr); - (*RecPtr) += pageHeaderSize; - targetRecOff = pageHeaderSize; - } - else if (targetRecOff < pageHeaderSize) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("invalid record offset at %X/%X", - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - goto next_record_is_invalid; - } - if ((((XLogPageHeader) readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) && - targetRecOff == pageHeaderSize) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("contrecord is requested by %X/%X", - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - goto next_record_is_invalid; - } - - /* - * Read the record length. - * - * NB: Even though we use an XLogRecord pointer here, the whole record - * header might not fit on this page. xl_tot_len is the first field of - * the struct, so it must be on this page (the records are MAXALIGNed), - * but we cannot access any other fields until we've verified that we - * got the whole header. - */ - record = (XLogRecord *) (readBuf + (*RecPtr) % XLOG_BLCKSZ); - total_len = record->xl_tot_len; - - /* - * If the whole record header is on this page, validate it immediately. - * Otherwise do just a basic sanity check on xl_tot_len, and validate the - * rest of the header after reading it from the next page. The xl_tot_len - * check is necessary here to ensure that we enter the "Need to reassemble - * record" code path below; otherwise we might fail to apply - * ValidXLogRecordHeader at all. - */ - if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) - { - if (!ValidXLogRecordHeader(RecPtr, record, emode, randAccess)) - goto next_record_is_invalid; - gotheader = true; - } - else - { - if (total_len < SizeOfXLogRecord) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("invalid record length at %X/%X", - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - goto next_record_is_invalid; - } - gotheader = false; - } + private->lastSourceFailed = false; - /* - * Allocate or enlarge readRecordBuf as needed. To avoid useless small - * increases, round its size to a multiple of XLOG_BLCKSZ, and make sure - * it's at least 4*Max(BLCKSZ, XLOG_BLCKSZ) to start with. (That is - * enough for all "normal" records, but very large commit or abort records - * might need more space.) - */ - if (total_len > readRecordBufSize) + do { - uint32 newSize = total_len; - - newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); - newSize = Max(newSize, 4 * Max(BLCKSZ, XLOG_BLCKSZ)); - if (readRecordBuf) - free(readRecordBuf); - readRecordBuf = (char *) malloc(newSize); - if (!readRecordBuf) + char *errormsg; + record = XLogReadRecord(xlogreader, RecPtr, &errormsg); + ReadRecPtr = xlogreader->ReadRecPtr; + EndRecPtr = xlogreader->EndRecPtr; + if (record == NULL) { - readRecordBufSize = 0; - /* We treat this as a "bogus data" condition */ - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("record length %u at %X/%X too long", - total_len, (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - goto next_record_is_invalid; - } - readRecordBufSize = newSize; - } - - len = XLOG_BLCKSZ - (*RecPtr) % XLOG_BLCKSZ; - if (total_len > len) - { - /* Need to reassemble record */ - char *contrecord; - XLogPageHeader pageHeader; - XLogRecPtr pagelsn; - char *buffer; - uint32 gotlen; - - /* Initialize pagelsn to the beginning of the page this record is on */ - pagelsn = ((*RecPtr) / XLOG_BLCKSZ) * XLOG_BLCKSZ; - - /* Copy the first fragment of the record from the first page. */ - memcpy(readRecordBuf, readBuf + (*RecPtr) % XLOG_BLCKSZ, len); - buffer = readRecordBuf + len; - gotlen = len; + ereport(emode_for_corrupt_record(xlogreader, RecPtr), + (errmsg_internal("%s", errormsg) /* already translated */)); - do - { - /* Calculate pointer to beginning of next page */ - XLByteAdvance(pagelsn, XLOG_BLCKSZ); - /* Wait for the next page to become available */ - if (!XLogPageRead(&pagelsn, emode, false, false)) - return NULL; - - /* Check that the continuation on next page looks valid */ - pageHeader = (XLogPageHeader) readBuf; - if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD)) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("there is no contrecord flag in log segment %s, offset %u", - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - goto next_record_is_invalid; - } - /* - * Cross-check that xlp_rem_len agrees with how much of the record - * we expect there to be left. - */ - if (pageHeader->xlp_rem_len == 0 || - total_len != (pageHeader->xlp_rem_len + gotlen)) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("invalid contrecord length %u in log segment %s, offset %u", - pageHeader->xlp_rem_len, - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - goto next_record_is_invalid; - } + private->lastSourceFailed = true; - /* Append the continuation from this page to the buffer */ - pageHeaderSize = XLogPageHeaderSize(pageHeader); - contrecord = (char *) readBuf + pageHeaderSize; - len = XLOG_BLCKSZ - pageHeaderSize; - if (pageHeader->xlp_rem_len < len) - len = pageHeader->xlp_rem_len; - memcpy(buffer, (char *) contrecord, len); - buffer += len; - gotlen += len; - - /* If we just reassembled the record header, validate it. */ - if (!gotheader) + if (private->readFile >= 0) { - record = (XLogRecord *) readRecordBuf; - if (!ValidXLogRecordHeader(RecPtr, record, emode, randAccess)) - goto next_record_is_invalid; - gotheader = true; + close(private->readFile); + private->readFile = -1; } - } while (pageHeader->xlp_rem_len > len); - - record = (XLogRecord *) readRecordBuf; - if (!RecordIsValid(record, *RecPtr, emode)) - goto next_record_is_invalid; - pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) readBuf); - XLogSegNoOffsetToRecPtr( - readSegNo, - readOff + pageHeaderSize + MAXALIGN(pageHeader->xlp_rem_len), - EndRecPtr); - ReadRecPtr = *RecPtr; - } - else - { - /* Record does not cross a page boundary */ - if (!RecordIsValid(record, *RecPtr, emode)) - goto next_record_is_invalid; - EndRecPtr = *RecPtr + MAXALIGN(total_len); - - ReadRecPtr = *RecPtr; - memcpy(readRecordBuf, record, total_len); - } - - /* - * Special processing if it's an XLOG SWITCH record - */ - if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH) - { - /* Pretend it extends to end of segment */ - EndRecPtr += XLogSegSize - 1; - EndRecPtr -= EndRecPtr % XLogSegSize; - - /* - * Pretend that readBuf contains the last page of the segment. This is - * just to avoid Assert failure in StartupXLOG if XLOG ends with this - * segment. - */ - readOff = XLogSegSize - XLOG_BLCKSZ; - } - return record; - -next_record_is_invalid: - lastSourceFailed = true; - - if (readFile >= 0) - { - close(readFile); - readFile = -1; - } - - /* In standby-mode, keep trying */ - if (StandbyMode) - goto retry; - else - return NULL; -} - -/* - * Check whether the xlog header of a page just read in looks valid. - * - * This is just a convenience subroutine to avoid duplicated code in - * ReadRecord. It's not intended for use from anywhere else. - */ -static bool -ValidXLogPageHeader(XLogPageHeader hdr, int emode, bool segmentonly) -{ - XLogRecPtr recaddr; - - XLogSegNoOffsetToRecPtr(readSegNo, readOff, recaddr); - - if (hdr->xlp_magic != XLOG_PAGE_MAGIC) - { - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("invalid magic number %04X in log segment %s, offset %u", - hdr->xlp_magic, - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - return false; - } - if ((hdr->xlp_info & ~XLP_ALL_FLAGS) != 0) - { - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("invalid info bits %04X in log segment %s, offset %u", - hdr->xlp_info, - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - return false; - } - if (hdr->xlp_info & XLP_LONG_HEADER) - { - XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr; - - if (longhdr->xlp_sysid != ControlFile->system_identifier) - { - char fhdrident_str[32]; - char sysident_str[32]; - - /* - * Format sysids separately to keep platform-dependent format code - * out of the translatable message string. - */ - snprintf(fhdrident_str, sizeof(fhdrident_str), UINT64_FORMAT, - longhdr->xlp_sysid); - snprintf(sysident_str, sizeof(sysident_str), UINT64_FORMAT, - ControlFile->system_identifier); - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("WAL file is from different database system"), - errdetail("WAL file database system identifier is %s, pg_control database system identifier is %s.", - fhdrident_str, sysident_str))); - return false; - } - if (longhdr->xlp_seg_size != XLogSegSize) - { - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("WAL file is from different database system"), - errdetail("Incorrect XLOG_SEG_SIZE in page header."))); - return false; - } - if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) - { - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("WAL file is from different database system"), - errdetail("Incorrect XLOG_BLCKSZ in page header."))); - return false; - } - } - else if (readOff == 0) - { - /* hmm, first page of file doesn't have a long header? */ - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("invalid info bits %04X in log segment %s, offset %u", - hdr->xlp_info, - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - return false; - } - - if (!XLByteEQ(hdr->xlp_pageaddr, recaddr)) - { - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("unexpected pageaddr %X/%X in log segment %s, offset %u", - (uint32) (hdr->xlp_pageaddr >> 32), (uint32) hdr->xlp_pageaddr, - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - return false; - } - - /* - * Check page TLI is one of the expected values. - */ - if (!tliInHistory(hdr->xlp_tli, expectedTLEs)) - { - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("unexpected timeline ID %u in log segment %s, offset %u", - hdr->xlp_tli, - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - return false; - } - - /* - * Since child timelines are always assigned a TLI greater than their - * immediate parent's TLI, we should never see TLI go backwards across - * successive pages of a consistent WAL sequence. - * - * Of course this check should only be applied when advancing sequentially - * across pages; therefore ReadRecord resets lastPageTLI and - * lastSegmentTLI to zero when going to a random page. - * - * Sometimes we re-open a segment that's already been partially replayed. - * In that case we cannot perform the normal TLI check: if there is a - * timeline switch within the segment, the first page has a smaller TLI - * than later pages following the timeline switch, and we might've read - * them already. As a weaker test, we still check that it's not smaller - * than the TLI we last saw at the beginning of a segment. Pass - * segmentonly = true when re-validating the first page like that, and the - * page you're actually interested in comes later. - */ - if (hdr->xlp_tli < (segmentonly ? lastSegmentTLI : lastPageTLI)) - { - ereport(emode_for_corrupt_record(emode, recaddr), - (errmsg("out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u", - hdr->xlp_tli, - segmentonly ? lastSegmentTLI : lastPageTLI, - XLogFileNameP(curFileTLI, readSegNo), - readOff))); - return false; - } - lastPageTLI = hdr->xlp_tli; - if (readOff == 0) - lastSegmentTLI = hdr->xlp_tli; - - return true; -} - -/* - * Validate an XLOG record header. - * - * This is just a convenience subroutine to avoid duplicated code in - * ReadRecord. It's not intended for use from anywhere else. - */ -static bool -ValidXLogRecordHeader(XLogRecPtr *RecPtr, XLogRecord *record, int emode, - bool randAccess) -{ - /* - * xl_len == 0 is bad data for everything except XLOG SWITCH, where it is - * required. - */ - if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH) - { - if (record->xl_len != 0) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("invalid xlog switch record at %X/%X", - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - return false; - } - } - else if (record->xl_len == 0) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("record with zero length at %X/%X", - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - return false; - } - if (record->xl_tot_len < SizeOfXLogRecord + record->xl_len || - record->xl_tot_len > SizeOfXLogRecord + record->xl_len + - XLR_MAX_BKP_BLOCKS * (sizeof(BkpBlock) + BLCKSZ)) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("invalid record length at %X/%X", - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - return false; - } - if (record->xl_rmid > RM_MAX_ID) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("invalid resource manager ID %u at %X/%X", - record->xl_rmid, (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - return false; - } - if (randAccess) - { - /* - * We can't exactly verify the prev-link, but surely it should be less - * than the record's own address. - */ - if (!XLByteLT(record->xl_prev, *RecPtr)) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("record with incorrect prev-link %X/%X at %X/%X", - (uint32) (record->xl_prev >> 32), (uint32) record->xl_prev, - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - return false; } - } - else - { - /* - * Record's prev-link should exactly match our previous location. This - * check guards against torn WAL pages where a stale but valid-looking - * WAL record starts on a sector boundary. - */ - if (!XLByteEQ(record->xl_prev, ReadRecPtr)) - { - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errmsg("record with incorrect prev-link %X/%X at %X/%X", - (uint32) (record->xl_prev >> 32), (uint32) record->xl_prev, - (uint32) ((*RecPtr) >> 32), (uint32) *RecPtr))); - return false; - } - } + } while (StandbyMode && record == NULL); - return true; + return record; } /* @@ -4792,7 +4223,8 @@ readRecoveryCommandFile(void) * Exit archive-recovery state */ static void -exitArchiveRecovery(TimeLineID endTLI, XLogSegNo endLogSegNo) +exitArchiveRecovery(XLogPageReadPrivate *private, TimeLineID endTLI, + XLogSegNo endLogSegNo) { char recoveryPath[MAXPGPATH]; char xlogpath[MAXPGPATH]; @@ -4811,10 +4243,10 @@ exitArchiveRecovery(TimeLineID endTLI, XLogSegNo endLogSegNo) * If the ending log segment is still open, close it (to avoid problems on * Windows with trying to rename or delete an open file). */ - if (readFile >= 0) + if (private->readFile >= 0) { - close(readFile); - readFile = -1; + close(private->readFile); + private->readFile = -1; } /* @@ -5255,6 +4687,8 @@ StartupXLOG(void) bool backupEndRequired = false; bool backupFromStandby = false; DBState dbstate_at_startup; + XLogReaderState *xlogreader; + XLogPageReadPrivate private; /* * Read control file and check XLOG status looks valid. @@ -5416,6 +4850,18 @@ StartupXLOG(void) if (StandbyMode) OwnLatch(&XLogCtl->recoveryWakeupLatch); + /* Set up XLOG reader facility */ + MemSet(&private, 0, sizeof(XLogPageReadPrivate)); + private.readFile = -1; + xlogreader = XLogReaderAllocate(InvalidXLogRecPtr, &XLogPageRead, &private); + if (!xlogreader) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed while allocating an XLog reading processor"))); + xlogreader->system_identifier = ControlFile->system_identifier; + xlogreader->expectedTLEs = expectedTLEs; + if (read_backup_label(&checkPointLoc, &backupEndRequired, &backupFromStandby)) { @@ -5423,7 +4869,7 @@ StartupXLOG(void) * When a backup_label file is present, we want to roll forward from * the checkpoint it identifies, rather than using pg_control. */ - record = ReadCheckpointRecord(checkPointLoc, 0); + record = ReadCheckpointRecord(xlogreader, checkPointLoc, 0); if (record != NULL) { memcpy(&checkPoint, XLogRecGetData(record), sizeof(CheckPoint)); @@ -5441,7 +4887,7 @@ StartupXLOG(void) */ if (XLByteLT(checkPoint.redo, checkPointLoc)) { - if (!ReadRecord(&(checkPoint.redo), LOG, false)) + if (!ReadRecord(xlogreader, checkPoint.redo, LOG, false)) ereport(FATAL, (errmsg("could not find redo location referenced by checkpoint record"), errhint("If you are not restoring from a backup, try removing the file \"%s/backup_label\".", DataDir))); @@ -5465,7 +4911,7 @@ StartupXLOG(void) */ checkPointLoc = ControlFile->checkPoint; RedoStartLSN = ControlFile->checkPointCopy.redo; - record = ReadCheckpointRecord(checkPointLoc, 1); + record = ReadCheckpointRecord(xlogreader, checkPointLoc, 1); if (record != NULL) { ereport(DEBUG1, @@ -5484,7 +4930,7 @@ StartupXLOG(void) else { checkPointLoc = ControlFile->prevCheckPoint; - record = ReadCheckpointRecord(checkPointLoc, 2); + record = ReadCheckpointRecord(xlogreader, checkPointLoc, 2); if (record != NULL) { ereport(LOG, @@ -5800,12 +5246,12 @@ StartupXLOG(void) if (XLByteLT(checkPoint.redo, RecPtr)) { /* back up to find the record */ - record = ReadRecord(&(checkPoint.redo), PANIC, false); + record = ReadRecord(xlogreader, checkPoint.redo, PANIC, false); } else { /* just have to read next record after CheckPoint */ - record = ReadRecord(NULL, LOG, false); + record = ReadRecord(xlogreader, InvalidXLogRecPtr, LOG, false); } if (record != NULL) @@ -5968,7 +5414,7 @@ StartupXLOG(void) break; /* Else, try to fetch the next WAL record */ - record = ReadRecord(NULL, LOG, false); + record = ReadRecord(xlogreader, InvalidXLogRecPtr, LOG, false); } while (record != NULL); /* @@ -6018,7 +5464,7 @@ StartupXLOG(void) * Re-fetch the last valid or last applied record, so we can identify the * exact endpoint of what we consider the valid portion of WAL. */ - record = ReadRecord(&LastRec, PANIC, false); + record = ReadRecord(xlogreader, LastRec, PANIC, false); EndOfLog = EndRecPtr; XLByteToPrevSeg(EndOfLog, endLogSegNo); @@ -6122,7 +5568,7 @@ StartupXLOG(void) * we will use that below.) */ if (InArchiveRecovery) - exitArchiveRecovery(curFileTLI, endLogSegNo); + exitArchiveRecovery(&private, xlogreader->readPageTLI, endLogSegNo); /* * Prepare to write WAL starting at EndOfLog position, and init xlog @@ -6141,8 +5587,15 @@ StartupXLOG(void) * record spans, not the one it starts in. The last block is indeed the * one we want to use. */ - Assert(readOff == (XLogCtl->xlblocks[0] - XLOG_BLCKSZ) % XLogSegSize); - memcpy((char *) Insert->currpage, readBuf, XLOG_BLCKSZ); + if (EndOfLog % XLOG_BLCKSZ == 0) + { + memset(Insert->currpage, 0, XLOG_BLCKSZ); + } + else + { + Assert(private.readOff == (XLogCtl->xlblocks[0] - XLOG_BLCKSZ) % XLogSegSize); + memcpy((char *) Insert->currpage, xlogreader->readBuf, XLOG_BLCKSZ); + } Insert->currpos = (char *) Insert->currpage + (EndOfLog + XLOG_BLCKSZ - XLogCtl->xlblocks[0]); @@ -6293,23 +5746,13 @@ StartupXLOG(void) if (standbyState != STANDBY_DISABLED) ShutdownRecoveryTransactionEnvironment(); - /* Shut down readFile facility, free space */ - if (readFile >= 0) + /* Shut down xlogreader */ + if (private.readFile >= 0) { - close(readFile); - readFile = -1; - } - if (readBuf) - { - free(readBuf); - readBuf = NULL; - } - if (readRecordBuf) - { - free(readRecordBuf); - readRecordBuf = NULL; - readRecordBufSize = 0; + close(private.readFile); + private.readFile = -1; } + XLogReaderFree(xlogreader); /* * If any of the critical GUCs have changed, log them before we allow @@ -6523,7 +5966,7 @@ LocalSetXLogInsertAllowed(void) * 1 for "primary", 2 for "secondary", 0 for "other" (backup_label) */ static XLogRecord * -ReadCheckpointRecord(XLogRecPtr RecPtr, int whichChkpt) +ReadCheckpointRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int whichChkpt) { XLogRecord *record; @@ -6547,7 +5990,7 @@ ReadCheckpointRecord(XLogRecPtr RecPtr, int whichChkpt) return NULL; } - record = ReadRecord(&RecPtr, LOG, true); + record = ReadRecord(xlogreader, RecPtr, LOG, true); if (record == NULL) { @@ -9311,28 +8754,24 @@ CancelBackup(void) * XLogPageRead() to try fetching the record from another source, or to * sleep and retry. */ -static bool -XLogPageRead(XLogRecPtr *RecPtr, int emode, bool fetching_ckpt, - bool randAccess) +static int +XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, + char *readBuf, TimeLineID *readTLI) { + XLogPageReadPrivate *private = + (XLogPageReadPrivate *) xlogreader->private_data; + int emode = private->emode; uint32 targetPageOff; - uint32 targetRecOff; XLogSegNo targetSegNo; - XLByteToSeg(*RecPtr, targetSegNo); - targetPageOff = (((*RecPtr) % XLogSegSize) / XLOG_BLCKSZ) * XLOG_BLCKSZ; - targetRecOff = (*RecPtr) % XLOG_BLCKSZ; - - /* Fast exit if we have read the record in the current buffer already */ - if (!lastSourceFailed && targetSegNo == readSegNo && - targetPageOff == readOff && targetRecOff < readLen) - return true; + XLByteToSeg(targetPagePtr, targetSegNo); + targetPageOff = targetPagePtr % XLogSegSize; /* * See if we need to switch to a new segment because the requested record * is not in the currently open one. */ - if (readFile >= 0 && !XLByteInSeg(*RecPtr, readSegNo)) + if (private->readFile >= 0 && !XLByteInSeg(targetPagePtr, private->readSegNo)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -9340,52 +8779,46 @@ XLogPageRead(XLogRecPtr *RecPtr, int emode, bool fetching_ckpt, */ if (StandbyMode && bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(private->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(private->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } - close(readFile); - readFile = -1; - readSource = 0; + close(private->readFile); + private->readFile = -1; + private->readSource = 0; } - XLByteToSeg(*RecPtr, readSegNo); + XLByteToSeg(targetPagePtr, private->readSegNo); retry: /* See if we need to retrieve more data */ - if (readFile < 0 || - (readSource == XLOG_FROM_STREAM && !XLByteLT(*RecPtr, receivedUpto))) + if (private->readFile < 0 || + (private->readSource == XLOG_FROM_STREAM && + !XLByteLT(targetPagePtr + reqLen, receivedUpto))) { if (StandbyMode) { - if (!WaitForWALToBecomeAvailable(*RecPtr, randAccess, - fetching_ckpt)) + if (!WaitForWALToBecomeAvailable(private, targetPagePtr + reqLen)) goto triggered; } - else + /* In archive or crash recovery. */ + else if (private->readFile < 0) { - /* In archive or crash recovery. */ - if (readFile < 0) - { - int source; - - /* Reset curFileTLI if random fetch. */ - if (randAccess) - curFileTLI = 0; + int source; - if (InArchiveRecovery) - source = XLOG_FROM_ANY; - else - source = XLOG_FROM_PG_XLOG; + if (InArchiveRecovery) + source = XLOG_FROM_ANY; + else + source = XLOG_FROM_PG_XLOG; - readFile = XLogFileReadAnyTLI(readSegNo, emode, source); - if (readFile < 0) - return false; - } + private->readFile = + XLogFileReadAnyTLI(private, private->readSegNo, emode, source); + if (private->readFile < 0) + return -1; } } @@ -9393,7 +8826,7 @@ retry: * At this point, we have the right segment open and if we're streaming we * know the requested record is in it. */ - Assert(readFile != -1); + Assert(private->readFile != -1); /* * If the current segment is being streamed from master, calculate how @@ -9401,98 +8834,72 @@ retry: * requested record has been received, but this is for the benefit of * future calls, to allow quick exit at the top of this function. */ - if (readSource == XLOG_FROM_STREAM) + if (private->readSource == XLOG_FROM_STREAM) { - if (((*RecPtr) / XLOG_BLCKSZ) != (receivedUpto / XLOG_BLCKSZ)) - { - readLen = XLOG_BLCKSZ; - } + if (((targetPagePtr) / XLOG_BLCKSZ) != (receivedUpto / XLOG_BLCKSZ)) + private->readLen = XLOG_BLCKSZ; else - readLen = receivedUpto % XLogSegSize - targetPageOff; + private->readLen = receivedUpto % XLogSegSize - targetPageOff; } else - readLen = XLOG_BLCKSZ; - - if (!readFileHeaderValidated && targetPageOff != 0) - { - /* - * Whenever switching to a new WAL segment, we read the first page of - * the file and validate its header, even if that's not where the - * target record is. This is so that we can check the additional - * identification info that is present in the first page's "long" - * header. - */ - readOff = 0; - if (read(readFile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - char fname[MAXFNAMELEN]; - XLogFileName(fname, curFileTLI, readSegNo); - ereport(emode_for_corrupt_record(emode, *RecPtr), - (errcode_for_file_access(), - errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); - goto next_record_is_invalid; - } - if (!ValidXLogPageHeader((XLogPageHeader) readBuf, emode, true)) - goto next_record_is_invalid; - } + private->readLen = XLOG_BLCKSZ; /* Read the requested page */ - readOff = targetPageOff; - if (lseek(readFile, (off_t) readOff, SEEK_SET) < 0) + private->readOff = targetPageOff; + if (lseek(private->readFile, (off_t) private->readOff, SEEK_SET) < 0) { - char fname[MAXFNAMELEN]; - XLogFileName(fname, curFileTLI, readSegNo); - ereport(emode_for_corrupt_record(emode, *RecPtr), + char fname[MAXFNAMELEN]; + + XLogFileName(fname, curFileTLI, private->readSegNo); + ereport(emode_for_corrupt_record(xlogreader, targetPagePtr + reqLen), (errcode_for_file_access(), - errmsg("could not seek in log segment %s to offset %u: %m", - fname, readOff))); + errmsg("could not seek in log segment %s to offset %u: %m", + fname, private->readOff))); goto next_record_is_invalid; } - if (read(readFile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + + if (read(private->readFile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { - char fname[MAXFNAMELEN]; - XLogFileName(fname, curFileTLI, readSegNo); - ereport(emode_for_corrupt_record(emode, *RecPtr), + char fname[MAXFNAMELEN]; + + XLogFileName(fname, curFileTLI, private->readSegNo); + ereport(emode_for_corrupt_record(xlogreader, targetPagePtr + reqLen), (errcode_for_file_access(), - errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + errmsg("could not read from log segment %s, offset %u: %m", + fname, private->readOff))); goto next_record_is_invalid; } - if (!ValidXLogPageHeader((XLogPageHeader) readBuf, emode, false)) - goto next_record_is_invalid; - readFileHeaderValidated = true; + Assert(targetSegNo == private->readSegNo); + Assert(targetPageOff == private->readOff); + Assert(reqLen <= private->readLen); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); - Assert(targetRecOff < readLen); - - return true; + *readTLI = curFileTLI; + return private->readLen; next_record_is_invalid: - lastSourceFailed = true; + private->lastSourceFailed = true; - if (readFile >= 0) - close(readFile); - readFile = -1; - readLen = 0; - readSource = 0; + if (private->readFile >= 0) + close(private->readFile); + private->readFile = -1; + private->readLen = 0; + private->readSource = 0; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; else - return false; + return -1; triggered: - if (readFile >= 0) - close(readFile); - readFile = -1; - readLen = 0; - readSource = 0; + if (private->readFile >= 0) + close(private->readFile); + private->readFile = -1; + private->readLen = 0; + private->readSource = 0; - return false; + return -1; } /* @@ -9507,8 +8914,7 @@ triggered: * false. */ static bool -WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt) +WaitForWALToBecomeAvailable(XLogPageReadPrivate *private, XLogRecPtr RecPtr) { static pg_time_t last_fail_time = 0; pg_time_t now; @@ -9534,12 +8940,12 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * part of advancing to the next state. *------- */ - if (currentSource == 0) - currentSource = XLOG_FROM_ARCHIVE; + if (private->currentSource == 0) + private->currentSource = XLOG_FROM_ARCHIVE; for (;;) { - int oldSource = currentSource; + int oldSource = private->currentSource; /* * First check if we failed to read from the current source, and @@ -9547,13 +8953,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * happened outside this function, e.g when a CRC check fails on a * record, or within this loop. */ - if (lastSourceFailed) + if (private->lastSourceFailed) { - switch (currentSource) + switch (private->currentSource) { case XLOG_FROM_ARCHIVE: - currentSource = XLOG_FROM_PG_XLOG; + private->currentSource = XLOG_FROM_PG_XLOG; break; case XLOG_FROM_PG_XLOG: @@ -9578,7 +8984,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, */ if (PrimaryConnInfo) { - XLogRecPtr ptr = fetching_ckpt ? RedoStartLSN : RecPtr; + XLogRecPtr ptr = private->fetching_ckpt ? RedoStartLSN : RecPtr; RequestXLogStreaming(ptr, PrimaryConnInfo); } @@ -9587,7 +8993,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * immediate failure if we didn't launch walreceiver, and * move on to the next state. */ - currentSource = XLOG_FROM_STREAM; + private->currentSource = XLOG_FROM_STREAM; break; case XLOG_FROM_STREAM: @@ -9620,7 +9026,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, { if (rescanLatestTimeLine()) { - currentSource = XLOG_FROM_ARCHIVE; + private->currentSource = XLOG_FROM_ARCHIVE; break; } } @@ -9639,60 +9045,61 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, now = (pg_time_t) time(NULL); } last_fail_time = now; - currentSource = XLOG_FROM_ARCHIVE; + private->currentSource = XLOG_FROM_ARCHIVE; break; default: - elog(ERROR, "unexpected WAL source %d", currentSource); + elog(ERROR, "unexpected WAL source %d", private->currentSource); } } - else if (currentSource == XLOG_FROM_PG_XLOG) + else if (private->currentSource == XLOG_FROM_PG_XLOG) { /* * We just successfully read a file in pg_xlog. We prefer files * in the archive over ones in pg_xlog, so try the next file * again from the archive first. */ - currentSource = XLOG_FROM_ARCHIVE; + private->currentSource = XLOG_FROM_ARCHIVE; } - if (currentSource != oldSource) + if (private->currentSource != oldSource) elog(DEBUG2, "switched WAL source from %s to %s after %s", - xlogSourceNames[oldSource], xlogSourceNames[currentSource], - lastSourceFailed ? "failure" : "success"); + xlogSourceNames[oldSource], + xlogSourceNames[private->currentSource], + private->lastSourceFailed ? "failure" : "success"); /* * We've now handled possible failure. Try to read from the chosen * source. */ - lastSourceFailed = false; + private->lastSourceFailed = false; - switch (currentSource) + switch (private->currentSource) { case XLOG_FROM_ARCHIVE: case XLOG_FROM_PG_XLOG: /* Close any old file we might have open. */ - if (readFile >= 0) + if (private->readFile >= 0) { - close(readFile); - readFile = -1; + close(private->readFile); + private->readFile = -1; } - /* Reset curFileTLI if random fetch. */ - if (randAccess) - curFileTLI = 0; /* * Try to restore the file from archive, or read an existing * file from pg_xlog. */ - readFile = XLogFileReadAnyTLI(readSegNo, DEBUG2, currentSource); - if (readFile >= 0) + private->readFile = + XLogFileReadAnyTLI(private, private->readSegNo, DEBUG2, + private->currentSource); + + if (private->readFile >= 0) return true; /* success! */ /* * Nope, not found in archive or pg_xlog. */ - lastSourceFailed = true; + private->lastSourceFailed = true; break; case XLOG_FROM_STREAM: @@ -9704,7 +9111,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, */ if (!WalRcvInProgress()) { - lastSourceFailed = true; + private->lastSourceFailed = true; break; } @@ -9745,17 +9152,18 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * open already. Use XLOG_FROM_STREAM so that source info * is set correctly and XLogReceiptTime isn't changed. */ - if (readFile < 0) + if (private->readFile < 0) { - readFile = XLogFileRead(readSegNo, PANIC, - recoveryTargetTLI, - XLOG_FROM_STREAM, false); - Assert(readFile >= 0); + private->readFile = + XLogFileRead(private, private->readSegNo, PANIC, + recoveryTargetTLI, + XLOG_FROM_STREAM, false); + Assert(private->readFile >= 0); } else { /* just make sure source info is correct... */ - readSource = XLOG_FROM_STREAM; + private->readSource = XLOG_FROM_STREAM; XLogReceiptSource = XLOG_FROM_STREAM; return true; } @@ -9776,7 +9184,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * will move on to replay the streamed WAL from pg_xlog, * and then recheck the trigger and exit replay. */ - lastSourceFailed = true; + private->lastSourceFailed = true; break; } @@ -9793,7 +9201,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, } default: - elog(ERROR, "unexpected WAL source %d", currentSource); + elog(ERROR, "unexpected WAL source %d", private->currentSource); } /* @@ -9825,11 +9233,13 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, * erroneously suppressed. */ static int -emode_for_corrupt_record(int emode, XLogRecPtr RecPtr) +emode_for_corrupt_record(XLogReaderState *state, XLogRecPtr RecPtr) { + XLogPageReadPrivate *private = (XLogPageReadPrivate *) state->private_data; + int emode = private->emode; static XLogRecPtr lastComplaint = 0; - if (readSource == XLOG_FROM_PG_XLOG && emode == LOG) + if (private->readSource == XLOG_FROM_PG_XLOG && emode == LOG) { if (XLByteEQ(RecPtr, lastComplaint)) emode = DEBUG1; diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c new file mode 100644 index 0000000..7ebb5bd --- /dev/null +++ b/src/backend/access/transam/xlogreader.c @@ -0,0 +1,992 @@ +/*------------------------------------------------------------------------- + * + * xlogreader.c + * Generic xlog reading facility + * + * Portions Copyright (c) 2012, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/transam/xlogreader.c + * + * NOTES + * Documentation about how do use this interface can be found in + * xlogreader.h, more specifically in the definition of the + * XLogReaderState struct where all parameters are documented. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/timeline.h" +#include "access/transam.h" +#include "access/xlog_internal.h" +#include "access/xlogreader.h" +#include "catalog/pg_control.h" + +static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); + +static bool ValidXLogPageHeader(XLogReaderState *state, XLogRecPtr recptr, + XLogPageHeader hdr); +static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, + XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess); +static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record, + XLogRecPtr recptr); +static int ReadPageInternal(struct XLogReaderState *state, XLogRecPtr pageptr, + int reqLen); +static void report_invalid_record(XLogReaderState *state, const char *fmt, ...) +/* This extension allows gcc to check the format string for consistency with + the supplied arguments. */ +__attribute__((format(PG_PRINTF_ATTRIBUTE, 2, 3))); + +/* size of the buffer allocated for error message. */ +#define MAX_ERRORMSG_LEN 1000 + +/* + * Construct a string in state->errormsg_buf explaining what's wrong with + * the current record being read. + */ +static void +report_invalid_record(XLogReaderState *state, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + vsnprintf(state->errormsg_buf, MAX_ERRORMSG_LEN, fmt, args); + va_end(args); +} + +/* + * Allocate and initialize a new xlog reader + * + * Returns NULL if the xlogreader couldn't be allocated. + */ +XLogReaderState * +XLogReaderAllocate(XLogRecPtr startpoint, XLogPageReadCB pagereadfunc, + void *private_data) +{ + XLogReaderState *state; + + state = (XLogReaderState *) malloc(sizeof(XLogReaderState)); + if (!state) + return NULL; + MemSet(state, 0, sizeof(XLogReaderState)); + + /* + * Permanently allocate readBuf. We do it this way, rather than just + * making a static array, for two reasons: (1) no need to waste the + * storage in most instantiations of the backend; (2) a static char array + * isn't guaranteed to have any particular alignment, whereas malloc() + * will provide MAXALIGN'd storage. + */ + state->readBuf = (char *) malloc(XLOG_BLCKSZ); + if (!state->readBuf) + { + free(state); + return NULL; + } + + state->read_page = pagereadfunc; + state->private_data = private_data; + state->EndRecPtr = startpoint; + state->readPageTLI = 0; + state->expectedTLEs = NIL; + state->system_identifier = 0; + state->errormsg_buf = malloc(MAX_ERRORMSG_LEN + 1); + if (!state->errormsg_buf) + { + free(state->readBuf); + free(state); + return NULL; + } + state->errormsg_buf[0] = '\0'; + + /* + * Allocate an initial readRecordBuf of minimal size, which can later be + * enlarged if necessary. + */ + if (!allocate_recordbuf(state, 0)) + { + free(state->errormsg_buf); + free(state->readBuf); + free(state); + return NULL; + } + + return state; +} + +void +XLogReaderFree(XLogReaderState *state) +{ + free(state->errormsg_buf); + if (state->readRecordBuf) + free(state->readRecordBuf); + free(state->readBuf); + free(state); +} + +/* + * Allocate readRecordBuf to fit a record of at least the given length. + * Returns true if successful, false if out of memory. + * + * readRecordBufSize is set to the new buffer size. + * + * To avoid useless small increases, round its size to a multiple of + * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start + * with. (That is enough for all "normal" records, but very large commit or + * abort records might need more space.) + */ +static bool +allocate_recordbuf(XLogReaderState *state, uint32 reclength) +{ + uint32 newSize = reclength; + + newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); + newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); + + if (state->readRecordBuf) + free(state->readRecordBuf); + state->readRecordBuf = (char *) malloc(newSize); + if (!state->readRecordBuf) + { + state->readRecordBufSize = 0; + return false; + } + + state->readRecordBufSize = newSize; + return true; +} + +/* + * Attempt to read an XLOG record. + * + * If RecPtr is not NULL, try to read a record at that position. Otherwise + * try to read a record just after the last one previously read. + * + * If no valid record is available, returns NULL. On NULL return, *errormsg + * is usually set to a string with details of the failure. One typical error + * where *errormsg is not set is when the read_page callback returns an error. + * + * The returned pointer (or *errormsg) points to an internal buffer that's + * valid until the next call to XLogReadRecord. + */ +XLogRecord * +XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) +{ + XLogRecord *record; + XLogRecPtr tmpRecPtr = state->EndRecPtr; + XLogRecPtr targetPagePtr; + bool randAccess = false; + uint32 len, + total_len; + uint32 targetRecOff; + uint32 pageHeaderSize; + bool gotheader; + int readOff; + + *errormsg = NULL; + state->errormsg_buf[0] = '\0'; + + if (RecPtr == InvalidXLogRecPtr) + { + RecPtr = tmpRecPtr; + + if (state->ReadRecPtr == InvalidXLogRecPtr) + randAccess = true; + + /* + * RecPtr is pointing to end+1 of the previous WAL record. If we're + * at a page boundary, no more records can fit on the current page. We + * must skip over the page header, but we can't do that until we've + * read in the page, since the header size is variable. + */ + } + else + { + /* + * In this case, the passed-in record pointer should already be + * pointing to a valid record starting position. + */ + Assert(XRecOffIsValid(RecPtr)); + randAccess = true; /* allow readPageTLI to go backwards too */ + } + + targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ); + + /* Read the page containing the record into state->readBuf */ + readOff = ReadPageInternal(state, targetPagePtr, SizeOfXLogRecord); + + if (readOff < 0) + { + if (state->errormsg_buf[0] != '\0') + *errormsg = state->errormsg_buf; + return NULL; + } + + /* ReadPageInternal always returns at least the page header */ + pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); + targetRecOff = RecPtr % XLOG_BLCKSZ; + if (targetRecOff == 0) + { + /* + * At page start, so skip over page header. + */ + RecPtr += pageHeaderSize; + targetRecOff = pageHeaderSize; + } + else if (targetRecOff < pageHeaderSize) + { + report_invalid_record(state, "invalid record offset at %X/%X", + (uint32) (RecPtr >> 32), (uint32) RecPtr); + *errormsg = state->errormsg_buf; + return NULL; + } + + if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) && + targetRecOff == pageHeaderSize) + { + report_invalid_record(state, "contrecord is requested by %X/%X", + (uint32) (RecPtr >> 32), (uint32) RecPtr); + *errormsg = state->errormsg_buf; + return NULL; + } + + /* ReadPageInternal has verified the page header */ + Assert(pageHeaderSize <= readOff); + + /* + * Ensure the whole record header or at least the part on this page is + * read. + */ + readOff = ReadPageInternal(state, + targetPagePtr, + Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)); + if (readOff < 0) + { + if (state->errormsg_buf[0] != '\0') + *errormsg = state->errormsg_buf; + return NULL; + } + + /* + * Read the record length. + * + * NB: Even though we use an XLogRecord pointer here, the whole record + * header might not fit on this page. xl_tot_len is the first field of the + * struct, so it must be on this page (the records are MAXALIGNed), but we + * cannot access any other fields until we've verified that we got the + * whole header. + */ + record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); + total_len = record->xl_tot_len; + + /* + * If the whole record header is on this page, validate it immediately. + * Otherwise do just a basic sanity check on xl_tot_len, and validate the + * rest of the header after reading it from the next page. The xl_tot_len + * check is necessary here to ensure that we enter the "Need to reassemble + * record" code path below; otherwise we might fail to apply + * ValidXLogRecordHeader at all. + */ + if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) + { + if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, record, + randAccess)) + { + if (state->errormsg_buf[0] != '\0') + *errormsg = state->errormsg_buf; + return NULL; + } + gotheader = true; + } + else + { + /* XXX: more validation should be done here */ + if (total_len < SizeOfXLogRecord) + { + report_invalid_record(state, "invalid record length at %X/%X", + (uint32) (RecPtr >> 32), (uint32) RecPtr); + *errormsg = state->errormsg_buf; + return NULL; + } + gotheader = false; + } + + /* + * Enlarge readRecordBuf as needed. + */ + if (total_len > state->readRecordBufSize && + !allocate_recordbuf(state, total_len)) + { + /* We treat this as a "bogus data" condition */ + report_invalid_record(state, "record length %u at %X/%X too long", + total_len, + (uint32) (RecPtr >> 32), (uint32) RecPtr); + *errormsg = state->errormsg_buf; + return NULL; + } + + len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ; + if (total_len > len) + { + /* Need to reassemble record */ + char *contdata; + XLogPageHeader pageHeader; + char *buffer; + uint32 gotlen; + + /* Copy the first fragment of the record from the first page. */ + memcpy(state->readRecordBuf, + state->readBuf + RecPtr % XLOG_BLCKSZ, len); + buffer = state->readRecordBuf + len; + gotlen = len; + + do + { + /* Calculate pointer to beginning of next page */ + XLByteAdvance(targetPagePtr, XLOG_BLCKSZ); + + /* Wait for the next page to become available */ + readOff = ReadPageInternal(state, targetPagePtr, + Min(len, XLOG_BLCKSZ)); + + if (readOff < 0) + goto err; + + Assert(SizeOfXLogShortPHD <= readOff); + + /* Check that the continuation on next page looks valid */ + pageHeader = (XLogPageHeader) state->readBuf; + if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD)) + { + report_invalid_record(state, + "there is no contrecord flag at %X/%X", + (uint32) (RecPtr >> 32), (uint32) RecPtr); + goto err; + } + + /* + * Cross-check that xlp_rem_len agrees with how much of the record + * we expect there to be left. + */ + if (pageHeader->xlp_rem_len == 0 || + total_len != (pageHeader->xlp_rem_len + gotlen)) + { + report_invalid_record(state, + "invalid contrecord length %u at %X/%X", + pageHeader->xlp_rem_len, + (uint32) (RecPtr >> 32), (uint32) RecPtr); + goto err; + } + + /* Append the continuation from this page to the buffer */ + pageHeaderSize = XLogPageHeaderSize(pageHeader); + Assert(pageHeaderSize <= readOff); + + contdata = (char *) state->readBuf + pageHeaderSize; + len = XLOG_BLCKSZ - pageHeaderSize; + if (pageHeader->xlp_rem_len < len) + len = pageHeader->xlp_rem_len; + + memcpy(buffer, (char *) contdata, len); + buffer += len; + gotlen += len; + + /* If we just reassembled the record header, validate it. */ + if (!gotheader) + { + record = (XLogRecord *) state->readRecordBuf; + if (!ValidXLogRecordHeader(state, RecPtr, state->ReadRecPtr, + record, randAccess)) + goto err; + gotheader = true; + } + } while (gotlen < total_len); + + Assert(gotheader); + + record = (XLogRecord *) state->readRecordBuf; + if (!ValidXLogRecord(state, record, RecPtr)) + goto err; + + pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); + state->ReadRecPtr = RecPtr; + state->EndRecPtr = targetPagePtr + pageHeaderSize + + MAXALIGN(pageHeader->xlp_rem_len); + } + else + { + /* Wait for the record data to become available */ + readOff = ReadPageInternal(state, targetPagePtr, + Min(targetRecOff + total_len, XLOG_BLCKSZ)); + if (readOff < 0) + goto err; + + /* Record does not cross a page boundary */ + if (!ValidXLogRecord(state, record, RecPtr)) + goto err; + + state->EndRecPtr = RecPtr + MAXALIGN(total_len); + + state->ReadRecPtr = RecPtr; + memcpy(state->readRecordBuf, record, total_len); + } + + /* + * Special processing if it's an XLOG SWITCH record + */ + if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH) + { + /* Pretend it extends to end of segment */ + state->EndRecPtr += XLogSegSize - 1; + state->EndRecPtr -= state->EndRecPtr % XLogSegSize; + } + + return record; + +err: + /* + * Invalidate the xlog page we've cached. We might read from a different + * source after failure. + */ + state->readSegNo = 0; + state->readOff = 0; + state->readLen = 0; + + if (state->errormsg_buf[0] != '\0') + *errormsg = state->errormsg_buf; + + return NULL; +} + +/* + * Find the first record with at an lsn >= RecPtr. + * + * Useful for checking wether RecPtr is a valid xlog address for reading and to + * find the first valid address after some address when dumping records for + * debugging purposes. + */ +XLogRecPtr +XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr) +{ + XLogReaderState saved_state = *state; + XLogRecPtr targetPagePtr; + XLogRecPtr tmpRecPtr; + int targetRecOff; + XLogRecPtr found = InvalidXLogRecPtr; + uint32 pageHeaderSize; + XLogPageHeader header; + XLogRecord *record; + uint32 readLen; + char *errormsg; + + if (RecPtr == InvalidXLogRecPtr) + RecPtr = state->EndRecPtr; + + targetRecOff = RecPtr % XLOG_BLCKSZ; + + /* scroll back to page boundary */ + targetPagePtr = RecPtr - targetRecOff; + + /* Read the page containing the record */ + readLen = ReadPageInternal(state, targetPagePtr, targetRecOff); + if (readLen < 0) + goto err; + + header = (XLogPageHeader) state->readBuf; + + pageHeaderSize = XLogPageHeaderSize(header); + + /* make sure we have enough data for the page header */ + readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize); + if (readLen < 0) + goto err; + + /* skip over potential continuation data */ + if (header->xlp_info & XLP_FIRST_IS_CONTRECORD) + { + /* record headers are MAXALIGN'ed */ + tmpRecPtr = targetPagePtr + pageHeaderSize + + MAXALIGN(header->xlp_rem_len); + } + else + { + tmpRecPtr = targetPagePtr + pageHeaderSize; + } + + /* + * we know now that tmpRecPtr is an address pointing to a valid XLogRecord + * because either were at the first record after the beginning of a page or + * we just jumped over the remaining data of a continuation. + */ + while ((record = XLogReadRecord(state, tmpRecPtr, &errormsg))) + { + /* continue after the record */ + tmpRecPtr = InvalidXLogRecPtr; + + /* past the record we've found, break out */ + if (XLByteLE(RecPtr, state->ReadRecPtr)) + { + found = state->ReadRecPtr; + goto out; + } + } + +err: +out: + /* Restore state to what we had before finding the record */ + saved_state.readRecordBuf = state->readRecordBuf; + saved_state.readRecordBufSize = state->readRecordBufSize; + *state = saved_state; + return found; +} + +/* + * Read a single xlog page including at least [pagestart, RecPtr] of valid data + * via the read_page() callback. + * + * Returns -1 if the required page cannot be read for some reason. + * + * We fetch the page from a reader-local cache if we know we have the required + * data and if there hasn't been any error since caching the data. + */ +static int +ReadPageInternal(struct XLogReaderState *state, XLogRecPtr pageptr, + int reqLen) +{ + int readLen; + uint32 targetPageOff; + XLogSegNo targetSegNo; + XLogPageHeader hdr; + + Assert((pageptr % XLOG_BLCKSZ) == 0); + + XLByteToSeg(pageptr, targetSegNo); + targetPageOff = (pageptr % XLogSegSize); + + /* check whether we have all the requested data already */ + if (targetSegNo == state->readSegNo && targetPageOff == state->readOff && + reqLen < state->readLen) + return state->readLen; + + /* + * Data is not cached. + * + * Everytime we actually read the page, even if we looked at parts of it + * before, we need to do verification as the read_page callback might now + * be rereading data from a different source. + * + * Whenever switching to a new WAL segment, we read the first page of the + * file and validate its header, even if that's not where the target record + * is. This is so that we can check the additional identification info + * that is present in the first page's "long" header. + */ + if (targetSegNo != state->readSegNo && + targetPageOff != 0) + { + XLogPageHeader hdr; + XLogRecPtr targetSegmentPtr = pageptr - targetPageOff; + + readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ, + state->readBuf, &state->readPageTLI); + + if (readLen < 0) + goto err; + + Assert(readLen <= XLOG_BLCKSZ); + + /* we can be sure to have enough WAL available, we scrolled back */ + Assert(readLen == XLOG_BLCKSZ); + + hdr = (XLogPageHeader) state->readBuf; + + if (!ValidXLogPageHeader(state, targetSegmentPtr, hdr)) + goto err; + } + + /* now read the target data */ + readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD), + state->readBuf, &state->readPageTLI); + if (readLen < 0) + goto err; + + Assert(readLen <= XLOG_BLCKSZ); + + /* check we have enough data to check for the actual length of a the page header */ + if (readLen <= SizeOfXLogShortPHD) + goto err; + + Assert(readLen >= reqLen); + + hdr = (XLogPageHeader) state->readBuf; + + /* still not enough */ + if (readLen < XLogPageHeaderSize(hdr)) + { + readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr), + state->readBuf, &state->readPageTLI); + if (readLen < 0) + goto err; + } + + if (!ValidXLogPageHeader(state, pageptr, hdr)) + goto err; + + /* update cache information */ + state->readSegNo = targetSegNo; + state->readOff = targetPageOff; + state->readLen = readLen; + + return readLen; +err: + state->readSegNo = 0; + state->readOff = 0; + state->readLen = 0; + return -1; +} + +/* + * Validate an XLOG record header. + * + * This is just a convenience subroutine to avoid duplicated code in + * XLogReadRecord. It's not intended for use from anywhere else. + */ +static bool +ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr, + XLogRecPtr PrevRecPtr, XLogRecord *record, + bool randAccess) +{ + /* + * xl_len == 0 is bad data for everything except XLOG SWITCH, where it is + * required. + */ + if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH) + { + if (record->xl_len != 0) + { + report_invalid_record(state, + "invalid xlog switch record at %X/%X", + (uint32) (RecPtr >> 32), (uint32) RecPtr); + return false; + } + } + else if (record->xl_len == 0) + { + report_invalid_record(state, + "record with zero length at %X/%X", + (uint32) (RecPtr >> 32), (uint32) RecPtr); + return false; + } + if (record->xl_tot_len < SizeOfXLogRecord + record->xl_len || + record->xl_tot_len > SizeOfXLogRecord + record->xl_len + + XLR_MAX_BKP_BLOCKS * (sizeof(BkpBlock) + BLCKSZ)) + { + report_invalid_record(state, + "invalid record length at %X/%X", + (uint32) (RecPtr >> 32), (uint32) RecPtr); + return false; + } + if (record->xl_rmid > RM_MAX_ID) + { + report_invalid_record(state, + "invalid resource manager ID %u at %X/%X", + record->xl_rmid, (uint32) (RecPtr >> 32), + (uint32) RecPtr); + return false; + } + if (randAccess) + { + /* + * We can't exactly verify the prev-link, but surely it should be less + * than the record's own address. + */ + if (!XLByteLT(record->xl_prev, RecPtr)) + { + report_invalid_record(state, + "record with incorrect prev-link %X/%X at %X/%X", + (uint32) (record->xl_prev >> 32), + (uint32) record->xl_prev, + (uint32) (RecPtr >> 32), (uint32) RecPtr); + return false; + } + } + else + { + /* + * Record's prev-link should exactly match our previous location. This + * check guards against torn WAL pages where a stale but valid-looking + * WAL record starts on a sector boundary. + */ + if (!XLByteEQ(record->xl_prev, PrevRecPtr)) + { + report_invalid_record(state, + "record with incorrect prev-link %X/%X at %X/%X", + (uint32) (record->xl_prev >> 32), + (uint32) record->xl_prev, + (uint32) (RecPtr >> 32), (uint32) RecPtr); + return false; + } + } + + return true; +} + + +/* + * CRC-check an XLOG record. We do not believe the contents of an XLOG + * record (other than to the minimal extent of computing the amount of + * data to read in) until we've checked the CRCs. + * + * We assume all of the record (that is, xl_tot_len bytes) has been read + * into memory at *record. Also, ValidXLogRecordHeader() has accepted the + * record's header, which means in particular that xl_tot_len is at least + * SizeOfXlogRecord, so it is safe to fetch xl_len. + */ +static bool +ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr) +{ + pg_crc32 crc; + int i; + uint32 len = record->xl_len; + BkpBlock bkpb; + char *blk; + size_t remaining = record->xl_tot_len; + + /* First the rmgr data */ + if (remaining < SizeOfXLogRecord + len) + { + /* ValidXLogRecordHeader() should've caught this already... */ + report_invalid_record(state, "invalid record length at %X/%X", + (uint32) (recptr >> 32), (uint32) recptr); + return false; + } + remaining -= SizeOfXLogRecord + len; + INIT_CRC32(crc); + COMP_CRC32(crc, XLogRecGetData(record), len); + + /* Add in the backup blocks, if any */ + blk = (char *) XLogRecGetData(record) + len; + for (i = 0; i < XLR_MAX_BKP_BLOCKS; i++) + { + uint32 blen; + + if (!(record->xl_info & XLR_BKP_BLOCK(i))) + continue; + + if (remaining < sizeof(BkpBlock)) + { + report_invalid_record(state, + "invalid backup block size in record at %X/%X", + (uint32) (recptr >> 32), (uint32) recptr); + return false; + } + memcpy(&bkpb, blk, sizeof(BkpBlock)); + + if (bkpb.hole_offset + bkpb.hole_length > BLCKSZ) + { + report_invalid_record(state, + "incorrect hole size in record at %X/%X", + (uint32) (recptr >> 32), (uint32) recptr); + return false; + } + blen = sizeof(BkpBlock) + BLCKSZ - bkpb.hole_length; + + if (remaining < blen) + { + report_invalid_record(state, + "invalid backup block size in record at %X/%X", + (uint32) (recptr >> 32), (uint32) recptr); + return false; + } + remaining -= blen; + COMP_CRC32(crc, blk, blen); + blk += blen; + } + + /* Check that xl_tot_len agrees with our calculation */ + if (remaining != 0) + { + report_invalid_record(state, + "incorrect total length in record at %X/%X", + (uint32) (recptr >> 32), (uint32) recptr); + return false; + } + + /* Finally include the record header */ + COMP_CRC32(crc, (char *) record, offsetof(XLogRecord, xl_crc)); + FIN_CRC32(crc); + + if (!EQ_CRC32(record->xl_crc, crc)) + { + report_invalid_record(state, + "incorrect resource manager data checksum in record at %X/%X", + (uint32) (recptr >> 32), (uint32) recptr); + return false; + } + + return true; +} + +static bool +ValidXLogPageHeader(XLogReaderState *state, XLogRecPtr recptr, + XLogPageHeader hdr) +{ + XLogRecPtr recaddr; + XLogSegNo segno; + int32 offset; + + Assert((recptr % XLOG_BLCKSZ) == 0); + + XLByteToSeg(recptr, segno); + offset = recptr % XLogSegSize; + + XLogSegNoOffsetToRecPtr(segno, offset, recaddr); + + if (hdr->xlp_magic != XLOG_PAGE_MAGIC) + { + char fname[MAXFNAMELEN]; + + XLogFileName(fname, state->readPageTLI, segno); + + report_invalid_record(state, + "invalid magic number %04X in log segment %s, offset %u", + hdr->xlp_magic, + fname, + offset); + return false; + } + + if ((hdr->xlp_info & ~XLP_ALL_FLAGS) != 0) + { + char fname[MAXFNAMELEN]; + + XLogFileName(fname, state->readPageTLI, segno); + + report_invalid_record(state, + "invalid info bits %04X in log segment %s, offset %u", + hdr->xlp_info, + fname, + offset); + return false; + } + + if (hdr->xlp_info & XLP_LONG_HEADER) + { + XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr; + + if (state->system_identifier && + longhdr->xlp_sysid != state->system_identifier) + { + char fhdrident_str[32]; + char sysident_str[32]; + + /* + * Format sysids separately to keep platform-dependent format code + * out of the translatable message string. + */ + snprintf(fhdrident_str, sizeof(fhdrident_str), UINT64_FORMAT, + longhdr->xlp_sysid); + snprintf(sysident_str, sizeof(sysident_str), UINT64_FORMAT, + state->system_identifier); + report_invalid_record(state, + "WAL file is from different database system: WAL file database system identifier is %s, pg_control database system identifier is %s.", + fhdrident_str, sysident_str); + return false; + } + else if (longhdr->xlp_seg_size != XLogSegSize) + { + report_invalid_record(state, + "WAL file is from different database system: Incorrect XLOG_SEG_SIZE in page header."); + return false; + } + else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) + { + report_invalid_record(state, + "WAL file is from different database system: Incorrect XLOG_BLCKSZ in page header."); + return false; + } + } + else if (offset == 0) + { + char fname[MAXFNAMELEN]; + + XLogFileName(fname, state->readPageTLI, segno); + + /* hmm, first page of file doesn't have a long header? */ + report_invalid_record(state, + "invalid info bits %04X in log segment %s, offset %u", + hdr->xlp_info, + fname, + offset); + return false; + } + + if (!XLByteEQ(hdr->xlp_pageaddr, recaddr)) + { + char fname[MAXFNAMELEN]; + + XLogFileName(fname, state->readPageTLI, segno); + + report_invalid_record(state, + "unexpected pageaddr %X/%X in log segment %s, offset %u", + (uint32) (hdr->xlp_pageaddr >> 32), (uint32) hdr->xlp_pageaddr, + fname, + offset); + return false; + } + + /* + * Check page TLI is one of the expected values. + */ + if (state->expectedTLEs != NIL && + !tliInHistory(hdr->xlp_tli, state->expectedTLEs)) + { + char fname[MAXFNAMELEN]; + + XLogFileName(fname, state->readPageTLI, segno); + + report_invalid_record(state, + "unexpected timeline ID %u in log segment %s, offset %u", + hdr->xlp_tli, + fname, + offset); + return false; + } + + /* + * Since child timelines are always assigned a TLI greater than their + * immediate parent's TLI, we should never see TLI go backwards across + * successive pages of a consistent WAL sequence. + * + * Of course this check should only be applied when advancing sequentially + * across pages; therefore ReadRecord resets lastPageTLI and lastSegmentTLI + * to zero when going to a random page. FIXME + * + * Sometimes we re-read a segment that's already been (partially) read. So + * we only verify TLIs for pages that are later than the last remembered + * LSN. + * + * XXX: This is slightly less precise than the check we did in earlier + * times. I don't see a problem with that though. + */ + if (state->latestReadPtr < recptr) + { + if (hdr->xlp_tli < state->latestReadTLI) + { + char fname[MAXFNAMELEN]; + + XLogFileName(fname, state->readPageTLI, segno); + + report_invalid_record(state, + "out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u", + hdr->xlp_tli, + state->latestReadTLI, + fname, + offset); + return false; + } + state->latestReadPtr = recptr; + state->latestReadTLI = hdr->xlp_tli; + } + return true; +} diff --git a/src/backend/nls.mk b/src/backend/nls.mk index 30f6a2b..0598e8f 100644 --- a/src/backend/nls.mk +++ b/src/backend/nls.mk @@ -4,12 +4,13 @@ AVAIL_LANGUAGES = de es fr ja pt_BR tr zh_CN zh_TW GETTEXT_FILES = + gettext-files GETTEXT_TRIGGERS = $(BACKEND_COMMON_GETTEXT_TRIGGERS) \ GUC_check_errmsg GUC_check_errdetail GUC_check_errhint \ - write_stderr yyerror parser_yyerror + write_stderr yyerror parser_yyerror report_corrupt_record GETTEXT_FLAGS = $(BACKEND_COMMON_GETTEXT_FLAGS) \ GUC_check_errmsg:1:c-format \ GUC_check_errdetail:1:c-format \ GUC_check_errhint:1:c-format \ - write_stderr:1:c-format + write_stderr:1:c-format \ + report_corrupt_record:2:c-format gettext-files: distprep find $(srcdir)/ $(srcdir)/../port/ -name '*.c' -print | LC_ALL=C sort >$@ diff --git a/src/bin/Makefile b/src/bin/Makefile index b4dfdba..86dace0 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -14,7 +14,7 @@ top_builddir = ../.. include $(top_builddir)/src/Makefile.global SUBDIRS = initdb pg_ctl pg_dump \ - psql scripts pg_config pg_controldata pg_resetxlog pg_basebackup + psql scripts pg_config pg_controldata pg_resetxlog pg_basebackup pg_xlogdump ifeq ($(PORTNAME), win32) SUBDIRS += pgevent diff --git a/src/bin/pg_xlogdump/Makefile b/src/bin/pg_xlogdump/Makefile new file mode 100644 index 0000000..ba126e9 --- /dev/null +++ b/src/bin/pg_xlogdump/Makefile @@ -0,0 +1,55 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/bin/pg_xlogdump +# +# Copyright (c) 1998-2012, PostgreSQL Global Development Group +# +# src/bin/pg_xlogdump/Makefile +# +#------------------------------------------------------------------------- + +PGFILEDESC = "pg_xlogdump" +PGAPPICON=win32 + +subdir = src/bin/pg_xlogdump +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +override CPPFLAGS := -DFRONTEND $(CPPFLAGS) + +all: pg_xlogdump + +xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% + rm -f $@ && $(LN_S) $< . + +assert.c: % : $(top_srcdir)/src/backend/utils/error/% + rm -f $@ && $(LN_S) $< . + +rmgrdescfiles = clogdesc.c dbasedesc.c gindesc.c gistdesc.c hashdesc.c \ + heapdesc.c mxactdesc.c nbtdesc.c relmapdesc.c seqdesc.c smgrdesc.c \ + spgdesc.c standbydesc.c tblspcdesc.c xactdesc.c xlogdesc.c + +$(rmgrdescfiles): % : $(top_srcdir)/src/backend/access/rmgrdesc/% + rm -f $@ && $(LN_S) $< . + +OBJS = \ + clogdesc.o dbasedesc.o gindesc.o gistdesc.o hashdesc.o heapdesc.o \ + mxactdesc.o nbtdesc.o relmapdesc.o seqdesc.o smgrdesc.o spgdesc.o \ + standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o assert.o \ + $(WIN32RES) \ + pg_xlogdump.o stdout_strinfo.o compat.o tables.o xlogreader.o \ + +pg_xlogdump: $(OBJS) | submake-libpgport + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) $(libpq_pgport) -o $@$(X) + +install: all installdirs + $(INSTALL_PROGRAM) pg_xlogdump$(X) '$(DESTDIR)$(bindir)/pg_xlogdump$(X)' + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(bindir)' + +uninstall: + rm -f '$(DESTDIR)$(bindir)/pg_xlogdump$(X)' + +clean distclean maintainer-clean: + rm -f $(OBJS) pg_xlogdump xlogreader.c assert.c $(rmgrdescfiles) diff --git a/src/bin/pg_xlogdump/compat.c b/src/bin/pg_xlogdump/compat.c new file mode 100644 index 0000000..dd36e55 --- /dev/null +++ b/src/bin/pg_xlogdump/compat.c @@ -0,0 +1,96 @@ +/*------------------------------------------------------------------------- + * + * compat.c + * Support functions for xlogdump.c + * + * Portions Copyright (c) 2012, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_xlogdump/compat.c + * + * This file contains client-side implementations for various backend + * functions that the rm_desc functions in *desc.c files rely on. + * + *------------------------------------------------------------------------- + */ + +/* ugly hack, same as in e.g pg_controldata */ +#define FRONTEND 1 +#include "postgres.h" + +#include "access/timeline.h" +#include "catalog/catalog.h" +#include "datatype/timestamp.h" +#include "storage/relfilenode.h" +#include "utils/timestamp.h" + +bool assert_enabled = false; + +/* + * Returns true if 'expectedTLEs' contains a timeline with id 'tli' + */ +bool +tliInHistory(TimeLineID tli, List *expectedTLEs) +{ + ListCell *cell; + + foreach(cell, expectedTLEs) + { + if (((TimeLineHistoryEntry *) lfirst(cell))->tli == tli) + return true; + } + + return false; +} + +void +pfree(void *a) +{ +} + + +const char * +timestamptz_to_str(TimestampTz t) +{ + return ""; +} + +char * +relpathbackend(RelFileNode rnode, BackendId backend, ForkNumber forknum) +{ + return NULL; +} + +/* + * Write errors to stderr (or by equal means when stderr is + * not available). + */ +void +write_stderr(const char *fmt,...) +{ + va_list ap; + + va_start(ap, fmt); +#if !defined(WIN32) && !defined(__CYGWIN__) + /* On Unix, we just fprintf to stderr */ + vfprintf(stderr, fmt, ap); +#else + + /* + * On Win32, we print to stderr if running on a console, or write to + * eventlog if running as a service + */ + if (!isatty(fileno(stderr))) /* Running as a service */ + { + char errbuf[2048]; /* Arbitrary size? */ + + vsnprintf(errbuf, sizeof(errbuf), fmt, ap); + + write_eventlog(EVENTLOG_ERROR_TYPE, errbuf); + } + else + /* Not running as service, write to stderr */ + vfprintf(stderr, fmt, ap); +#endif + va_end(ap); +} diff --git a/src/bin/pg_xlogdump/nls.mk b/src/bin/pg_xlogdump/nls.mk new file mode 100644 index 0000000..3a981f5 --- /dev/null +++ b/src/bin/pg_xlogdump/nls.mk @@ -0,0 +1,4 @@ +# src/bin/pg_xlogdump/nls.mk +CATALOG_NAME = pg_xlogdump +AVAIL_LANGUAGES = +GETTEXT_FILES = pg_xlogdump.c diff --git a/src/bin/pg_xlogdump/pg_xlogdump.c b/src/bin/pg_xlogdump/pg_xlogdump.c new file mode 100644 index 0000000..6b24da6 --- /dev/null +++ b/src/bin/pg_xlogdump/pg_xlogdump.c @@ -0,0 +1,455 @@ +/*------------------------------------------------------------------------- + * + * pg_xlogdump.c - decode and display WAL + * + * Copyright (c) 2012, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_xlogdump/pg_xlogdump.c + *------------------------------------------------------------------------- + */ + +/* ugly hack, same as in e.g pg_controldata */ +#define FRONTEND 1 +#include "postgres.h" + +#include +#include + +#include "access/xlogreader.h" +#include "access/rmgr.h" +#include "catalog/catalog.h" +#include "pg_config_manual.h" +#include "utils/elog.h" + +#include "getopt_long.h" + +static const char *progname; + +typedef struct XLogDumpPrivateData +{ + TimeLineID timeline; + char *outpath; + char *inpath; + char *file; + XLogRecPtr startptr; + XLogRecPtr endptr; + + bool bkp_details; +} XLogDumpPrivateData; + +static void fatal_error(const char *fmt, ...) +__attribute__((format(PG_PRINTF_ATTRIBUTE, 1, 2))); + +static void fatal_error(const char *fmt, ...) +{ + va_list args; + fflush(stdout); + + fprintf(stderr, "fatal_error: "); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fprintf(stderr, "\n"); + exit(1); +} + +static void +XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, + XLogRecPtr startptr, char *buf, Size count); + +/* this should probably be put in a general implementation */ +static void +XLogDumpXLogRead(const char *directory, TimeLineID timeline_id, + XLogRecPtr startptr, char *buf, Size count) +{ + char *p; + XLogRecPtr recptr; + Size nbytes; + + static int sendFile = -1; + static XLogSegNo sendSegNo = 0; + static uint32 sendOff = 0; + + p = buf; + recptr = startptr; + nbytes = count; + + while (nbytes > 0) + { + uint32 startoff; + int segbytes; + int readbytes; + + startoff = recptr % XLogSegSize; + + if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo)) + { + char fname[MAXFNAMELEN]; + char fpath[MAXPGPATH]; + + /* Switch to another logfile segment */ + if (sendFile >= 0) + close(sendFile); + + XLByteToSeg(recptr, sendSegNo); + + XLogFileName(fname, timeline_id, sendSegNo); + + snprintf(fpath, MAXPGPATH, "%s/%s", + (directory == NULL) ? XLOGDIR : directory, fname); + + sendFile = open(fpath, O_RDONLY, 0); + if (sendFile < 0) + { + /* + * If the file is not found, assume it's because the standby + * asked for a too old WAL segment that has already been + * removed or recycled. + */ + if (errno == ENOENT) + fatal_error("requested WAL segment %s has already been removed", + fname); + else + fatal_error("could not open file \"%s\": %u", + fpath, errno); + } + sendOff = 0; + } + + /* Need to seek in the file? */ + if (sendOff != startoff) + { + if (lseek(sendFile, (off_t) startoff, SEEK_SET) < 0){ + char fname[MAXPGPATH]; + XLogFileName(fname, timeline_id, sendSegNo); + + fatal_error("could not seek in log segment %s to offset %u: %d", + fname, + startoff, + errno); + } + sendOff = startoff; + } + + /* How many bytes are within this segment? */ + if (nbytes > (XLogSegSize - startoff)) + segbytes = XLogSegSize - startoff; + else + segbytes = nbytes; + + readbytes = read(sendFile, p, segbytes); + if (readbytes <= 0) + { + char fname[MAXPGPATH]; + XLogFileName(fname, timeline_id, sendSegNo); + + fatal_error("could not read from log segment %s, offset %u, length %lu: %d", + fname, + sendOff, (unsigned long) segbytes, errno); + } + + /* Update state for read */ + XLByteAdvance(recptr, readbytes); + + sendOff += readbytes; + nbytes -= readbytes; + p += readbytes; + } +} + +static int +XLogDumpReadPage(XLogReaderState* state, XLogRecPtr targetPagePtr, int reqLen, + char *readBuff, TimeLineID *curFileTLI) +{ + XLogDumpPrivateData *private = state->private_data; + int count = XLOG_BLCKSZ; + + if (private->endptr != InvalidXLogRecPtr) + { + if (targetPagePtr > private->endptr) + return -1; + + if (targetPagePtr + reqLen > private->endptr) + count = private->endptr - targetPagePtr; + } + + XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr, + readBuff, count); + + return count; +} + +static void +XLogDumpDisplayRecord(XLogReaderState* state, XLogRecord* record) +{ + XLogDumpPrivateData *config = (XLogDumpPrivateData *)state->private_data; + const RmgrData *rmgr = &RmgrTable[record->xl_rmid]; + + fprintf(stdout, "xlog record: rmgr: %-11s, record_len: %6u, tot_len: %6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, bkp: %u%u%u%u, desc:", + rmgr->rm_name, + record->xl_len, record->xl_tot_len, + record->xl_xid, + (uint32) (state->ReadRecPtr >> 32), (uint32) state->ReadRecPtr, + (uint32) (record->xl_prev >> 32), (uint32) record->xl_prev, + !!(XLR_BKP_BLOCK(0) & record->xl_info), + !!(XLR_BKP_BLOCK(1) & record->xl_info), + !!(XLR_BKP_BLOCK(2) & record->xl_info), + !!(XLR_BKP_BLOCK(3) & record->xl_info)); + + /* the desc routine will printf the description directly to stdout */ + rmgr->rm_desc(NULL, record->xl_info, XLogRecGetData(record)); + + fprintf(stdout, "\n"); + + if (config->bkp_details) + { + int off; + char *blk = (char *) XLogRecGetData(record) + record->xl_len; + + for (off = 0; off < XLR_MAX_BKP_BLOCKS; off++) + { + BkpBlock bkpb; + + if (!(XLR_BKP_BLOCK(off) & record->xl_info)) + continue; + + memcpy(&bkpb, blk, sizeof(BkpBlock)); + blk += sizeof(BkpBlock); + + fprintf(stdout, "\tbackup bkp #%u; rel %u/%u/%u; fork: %s; block: %u; hole: offset: %u, length: %u\n", + off, bkpb.node.spcNode, bkpb.node.dbNode, bkpb.node.relNode, + forkNames[bkpb.fork], bkpb.block, bkpb.hole_offset, bkpb.hole_length); + } + } +} + +static void +usage(const char *progname) +{ + printf(_("%s: reads/writes postgres transaction logs for debugging.\n\n"), + progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]...\n"), progname); + printf(_("\nOptions:\n")); + printf(_(" -b, --bkp-details output detailed information about backup blocks\n")); + printf(_(" -e, --end RECPTR read wal up to RECPTR\n")); + printf(_(" -f, --file FILE wal file to parse, cannot be specified together with -p\n")); + printf(_(" -h, --help show this help, then exit\n")); + printf(_(" -p, --path PATH from where do we want to read? cwd/pg_xlog is the default\n")); + printf(_(" -s, --start RECPTR read wal in directory indicated by -p starting at RECPTR\n")); + printf(_(" -t, --timeline TLI which timeline do we want to read, defaults to 1\n")); + printf(_(" -v, --version output version information, then exit\n")); +} + +int +main(int argc, char **argv) +{ + uint32 xlogid; + uint32 xrecoff; + XLogReaderState *xlogreader_state; + XLogDumpPrivateData private; + XLogRecord *record; + + static struct option long_options[] = { + {"bkp-details", no_argument, NULL, 'b'}, + {"end", required_argument, NULL, 'e'}, + {"file", required_argument, NULL, 'f'}, + {"help", no_argument, NULL, '?'}, + {"path", required_argument, NULL, 'p'}, + {"start", required_argument, NULL, 's'}, + {"timeline", required_argument, NULL, 't'}, + {"version", no_argument, NULL, 'V'}, + {NULL, 0, NULL, 0} + }; + + int c; + int option_index; + + + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_xlogdump")); + + progname = get_progname(argv[0]); + + + memset(&private, 0, sizeof(XLogDumpPrivateData)); + + private.timeline = 1; + private.bkp_details = false; + private.startptr = InvalidXLogRecPtr; + private.endptr = InvalidXLogRecPtr; + + if (argc <= 1) + { + fprintf(stderr, _("%s: no arguments specified\n"), progname); + goto bad_argument; + } + + while ((c = getopt_long(argc, argv, "be:f:hp:s:t:V", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'b': + private.bkp_details = true; + break; + case 'e': + if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2) + { + fprintf(stderr, _("%s: couldn't parse -e %s\n"), + progname, optarg); + goto bad_argument; + } + else + private.endptr = (uint64)xlogid << 32 | xrecoff; + break; + case 'f': + private.file = strdup(optarg); + break; + case '?': + usage(progname); + exit(0); + break; + case 'p': + private.inpath = strdup(optarg); + break; + case 's': + if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2) + { + fprintf(stderr, _("%s: couldn't parse -s %s\n"), + progname, optarg); + goto bad_argument; + } + else + private.startptr = (uint64)xlogid << 32 | xrecoff; + break; + case 't': + if (sscanf(optarg, "%d", &private.timeline) != 1) + { + fprintf(stderr, _("%s: couldn't parse timeline -t %s\n"), + progname, optarg); + goto bad_argument; + } + break; + case 'V': + printf("%s: (PostgreSQL): %s\n", progname, PG_VERSION); + exit(0); + break; + default: + fprintf(stderr, _("%s: unknown argument -%c passed\n"), + progname, c); + goto bad_argument; + break; + } + } + + /* some parameter was badly specified, don't output further errors */ + if (optind < argc) + { + fprintf(stderr, + _("%s: too many command-line arguments (first is \"%s\")\n"), + progname, argv[optind]); + goto bad_argument; + } + else if (private.inpath != NULL && private.file != NULL) + { + fprintf(stderr, + _("%s: only one of -p or -f can be specified\n"), + progname); + goto bad_argument; + } + /* no file specified, but no range of of interesting data either */ + else if (private.file == NULL && XLByteEQ(private.startptr, InvalidXLogRecPtr)) + { + fprintf(stderr, + _("%s: no -s given in range mode.\n"), + progname); + goto bad_argument; + } + /* everything ok, do some more setup */ + else + { + /* default value */ + if (private.file == NULL && private.inpath == NULL) + private.inpath = "pg_xlog"; + + /* XXX: validate directory */ + + /* default value */ + if (private.file != NULL) + { + XLogSegNo segno; + + /* FIXME: can we rely on basename? */ + XLogFromFileName(basename(private.file), &private.timeline, &segno); + private.inpath = strdup(dirname(private.file)); + + if (XLByteEQ(private.startptr, InvalidXLogRecPtr)) + XLogSegNoOffsetToRecPtr(segno, 0, private.startptr); + else if (!XLByteInSeg(private.startptr, segno)) + { + fprintf(stderr, + _("%s: -s does not lie inside file \"%s\"\n"), + progname, + private.file); + goto bad_argument; + } + + if (XLByteEQ(private.endptr, InvalidXLogRecPtr)) + XLogSegNoOffsetToRecPtr(segno + 1, 0, private.endptr); + else if (!XLByteInSeg(private.endptr, segno) && + private.endptr != (segno + 1) * XLogSegSize) + { + fprintf(stderr, + _("%s: -e does not lie inside file \"%s\"\n"), + progname, private.file); + goto bad_argument; + } + } + } + + /* we have everything we need, continue */ + { + XLogRecPtr first_record; + char *errormsg; + + xlogreader_state = XLogReaderAllocate(private.startptr, + XLogDumpReadPage, + &private); + + /* first find a valid recptr to start from */ + first_record = XLogFindNextRecord(xlogreader_state, private.startptr); + + if (first_record == InvalidXLogRecPtr) + fatal_error("Could not find a valid record after %X/%X", + (uint32) (private.startptr >> 32), + (uint32) private.startptr); + + /* + * Display a message that were skipping data if `from` wasn't a pointer + * to the start of a record and also wasn't a pointer to the beginning + * of a segment (e.g. we were used in file mode). + */ + if (first_record != private.startptr && (private.startptr % XLogSegSize) != 0) + fprintf(stdout, "first record is after %X/%X, at %X/%X, skipping over %u bytes\n", + (uint32) (private.startptr >> 32), (uint32) private.startptr, + (uint32) (first_record >> 32), (uint32) first_record, + (uint32) (first_record - private.endptr)); + + while ((record = XLogReadRecord(xlogreader_state, first_record, &errormsg))) + { + /* continue after the last record */ + first_record = InvalidXLogRecPtr; + XLogDumpDisplayRecord(xlogreader_state, record); + } + if (errormsg) + fprintf(stderr, "error in WAL record: %s\n", errormsg); + + XLogReaderFree(xlogreader_state); + } + + return 0; +bad_argument: + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); +} diff --git a/src/bin/pg_xlogdump/stdout_strinfo.c b/src/bin/pg_xlogdump/stdout_strinfo.c new file mode 100644 index 0000000..07e6d84 --- /dev/null +++ b/src/bin/pg_xlogdump/stdout_strinfo.c @@ -0,0 +1,23 @@ +/* + * A client-side StringInfo implementation that just prints everything to + * stdout + */ +#include "postgres_fe.h" + +#include "lib/stringinfo.h" + +void +appendStringInfo(StringInfo str, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); +} + +void +appendStringInfoString(StringInfo str, const char *string) +{ + appendStringInfo(str, "%s", string); +} diff --git a/src/bin/pg_xlogdump/tables.c b/src/bin/pg_xlogdump/tables.c new file mode 100644 index 0000000..b3a7dca --- /dev/null +++ b/src/bin/pg_xlogdump/tables.c @@ -0,0 +1,78 @@ +/*------------------------------------------------------------------------- + * + * tables.c + * Support data for xlogdump.c + * + * Portions Copyright (c) 2012, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_xlogdump/tables.c + * + * NOTES + * + *------------------------------------------------------------------------- + */ + +/* + * rmgr.c + * + * Resource managers definition + * + * src/backend/access/transam/rmgr.c + */ +#include "postgres.h" + +#include "access/clog.h" +#include "access/gin.h" +#include "access/gist_private.h" +#include "access/hash.h" +#include "access/heapam_xlog.h" +#include "access/multixact.h" +#include "access/nbtree.h" +#include "access/spgist.h" +#include "access/xact.h" +#include "access/xlog_internal.h" +#include "catalog/storage_xlog.h" +#include "commands/dbcommands.h" +#include "commands/sequence.h" +#include "commands/tablespace.h" +#include "storage/standby.h" +#include "utils/relmapper.h" +#include "catalog/catalog.h" + +/* + * Table of fork names. + * + * needs to be synced with src/backend/catalog/catalog.c + */ +const char *forkNames[] = { + "main", /* MAIN_FORKNUM */ + "fsm", /* FSM_FORKNUM */ + "vm", /* VISIBILITYMAP_FORKNUM */ + "init" /* INIT_FORKNUM */ +}; + +/* + * RmgrTable linked only to functions available outside of the backend. + * + * needs to be synced with src/backend/access/transam/rmgr.c + */ +const RmgrData RmgrTable[RM_MAX_ID + 1] = { + {"XLOG", NULL, xlog_desc, NULL, NULL, NULL}, + {"Transaction", NULL, xact_desc, NULL, NULL, NULL}, + {"Storage", NULL, smgr_desc, NULL, NULL, NULL}, + {"CLOG", NULL, clog_desc, NULL, NULL, NULL}, + {"Database", NULL, dbase_desc, NULL, NULL, NULL}, + {"Tablespace", NULL, tblspc_desc, NULL, NULL, NULL}, + {"MultiXact", NULL, multixact_desc, NULL, NULL, NULL}, + {"RelMap", NULL, relmap_desc, NULL, NULL, NULL}, + {"Standby", NULL, standby_desc, NULL, NULL, NULL}, + {"Heap2", NULL, heap2_desc, NULL, NULL, NULL}, + {"Heap", NULL, heap_desc, NULL, NULL, NULL}, + {"Btree", NULL, btree_desc, NULL, NULL, NULL}, + {"Hash", NULL, hash_desc, NULL, NULL, NULL}, + {"Gin", NULL, gin_desc, NULL, NULL, NULL}, + {"Gist", NULL, gist_desc, NULL, NULL, NULL}, + {"Sequence", NULL, seq_desc, NULL, NULL, NULL}, + {"SPGist", NULL, spg_desc, NULL, NULL, NULL} +}; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h new file mode 100644 index 0000000..6a1c060 --- /dev/null +++ b/src/include/access/xlogreader.h @@ -0,0 +1,136 @@ +/*------------------------------------------------------------------------- + * + * readxlog.h + * + * Generic xlog reading facility. + * + * Portions Copyright (c) 2012, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/include/access/xlogreader.h + * + * NOTES + * Check the definition of the XLogReaderState struct for instructions on + * how to use the XLogReader infrastructure. + * + * The basic idea is to allocate an XLogReaderState via + * XLogReaderAllocate, and call XLogReadRecord() until it returns NULL. + *------------------------------------------------------------------------- + */ +#ifndef XLOGREADER_H +#define XLOGREADER_H + +#include "access/xlog_internal.h" +#include "nodes/pg_list.h" + +struct XLogReaderState; + +/* + * The callbacks are explained in more detail inside the XLogReaderState + * struct. + */ + +typedef int (*XLogPageReadCB) (struct XLogReaderState *state, + XLogRecPtr pageptr, + int reqLen, + char *readBuf, + TimeLineID *pageTLI); + +typedef struct XLogReaderState +{ + /* ---------------------------------------- + * Public parameters + * ---------------------------------------- + */ + + /* + * Data input callback (mandatory). + * + * This callback shall read the the xlog page (of size XLOG_BLKSZ) in which + * RecPtr resides. All data <= RecPtr must be visible. The callback shall + * return the range of actually valid bytes returned or -1 upon + * failure. + * + * *pageTLI should be set to the TLI of the file the page was read from + * to be in. It is currently used only for error reporting purposes, to + * reconstruct the name of the WAL file where an error occurred. + */ + XLogPageReadCB read_page; + + /* + * System identifier of the xlog files were about to read. + * + * Set to zero (the default value) if unknown or unimportant. + */ + uint64 system_identifier; + + /* + * List of acceptable TLIs. + * + * Set to NIL (the default value) if this should not be checked. + */ + List *expectedTLEs; + + /* + * Opaque data for callbacks to use. Not used by XLogReader. + */ + void *private_data; + + /* + * From where to where are we reading + */ + XLogRecPtr ReadRecPtr; /* start of last record read */ + XLogRecPtr EndRecPtr; /* end+1 of last record read */ + + /* ---------------------------------------- + * private/internal state + * ---------------------------------------- + */ + + /* Buffer for currently read page (XLOG_BLCKSZ bytes) */ + char *readBuf; + + /* last read segment, segment offset, read length, TLI */ + XLogSegNo readSegNo; + uint32 readOff; + uint32 readLen; + TimeLineID readPageTLI; + + /* Highest TLI we have read so far */ + TimeLineID latestReadTLI; + XLogRecPtr latestReadPtr; + + /* Buffer for current ReadRecord result (expandable) */ + char *readRecordBuf; + uint32 readRecordBufSize; + + /* Buffer to hold error message */ + char *errormsg_buf; +} XLogReaderState; + +/* + * Get a new XLogReader + * + * At least the read_page callback, startptr and endptr have to be set before + * the reader can be used. + */ +extern XLogReaderState *XLogReaderAllocate(XLogRecPtr startpoint, + XLogPageReadCB pagereadfunc, void *private_data); + +/* + * Free an XLogReader + */ +extern void XLogReaderFree(XLogReaderState *state); + +/* + * Read the next record from xlog. Returns NULL on end-of-WAL or on failure. + */ +extern XLogRecord *XLogReadRecord(XLogReaderState *state, XLogRecPtr ptr, + char **errormsg); + +/* + * Find the address of the next record with a lsn >= RecPtr. + */ +extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr); + +#endif /* XLOGREADER_H */