Writing WAL for relcache invalidation: pg_internal.init
Enclose a patch for new WAL records for relcache invalidation.
Not conclusively tested, but seems to work fine, so please regard this
as a prototype-needs-review.
We definitely need a regr test framework to support this type of patch.
--
Simon Riggs
EnterpriseDB http://www.enterprisedb.com
Attachments:
xlog_relcache.patchtext/x-patch; charset=UTF-8; name=xlog_relcache.patchDownload
Index: src/backend/access/transam/rmgr.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/access/transam/rmgr.c,v
retrieving revision 1.24
diff -c -r1.24 rmgr.c
*** src/backend/access/transam/rmgr.c 7 Aug 2006 16:57:56 -0000 1.24
--- src/backend/access/transam/rmgr.c 27 Oct 2006 20:36:09 -0000
***************
*** 20,25 ****
--- 20,26 ----
#include "commands/sequence.h"
#include "commands/tablespace.h"
#include "storage/smgr.h"
+ #include "utils/relcache.h"
const RmgrData RmgrTable[RM_MAX_ID + 1] = {
***************
*** 30,36 ****
{"Database", dbase_redo, dbase_desc, NULL, NULL, NULL},
{"Tablespace", tblspc_redo, tblspc_desc, NULL, NULL, NULL},
{"MultiXact", multixact_redo, multixact_desc, NULL, NULL, NULL},
! {"Reserved 7", NULL, NULL, NULL, NULL, NULL},
{"Reserved 8", NULL, NULL, NULL, NULL, NULL},
{"Reserved 9", NULL, NULL, NULL, NULL, NULL},
{"Heap", heap_redo, heap_desc, NULL, NULL, NULL},
--- 31,37 ----
{"Database", dbase_redo, dbase_desc, NULL, NULL, NULL},
{"Tablespace", tblspc_redo, tblspc_desc, NULL, NULL, NULL},
{"MultiXact", multixact_redo, multixact_desc, NULL, NULL, NULL},
! {"Cache", cache_redo, cache_desc, NULL, NULL, NULL},
{"Reserved 8", NULL, NULL, NULL, NULL, NULL},
{"Reserved 9", NULL, NULL, NULL, NULL, NULL},
{"Heap", heap_redo, heap_desc, NULL, NULL, NULL},
Index: src/backend/access/transam/xlog.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/access/transam/xlog.c,v
retrieving revision 1.252
diff -c -r1.252 xlog.c
*** src/backend/access/transam/xlog.c 18 Oct 2006 22:44:11 -0000 1.252
--- src/backend/access/transam/xlog.c 27 Oct 2006 20:36:19 -0000
***************
*** 46,51 ****
--- 46,52 ----
#include "utils/builtins.h"
#include "utils/nabstime.h"
#include "utils/pg_locale.h"
+ #include "utils/relcache.h"
/*
Index: src/backend/utils/cache/relcache.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/utils/cache/relcache.c,v
retrieving revision 1.249
diff -c -r1.249 relcache.c
*** src/backend/utils/cache/relcache.c 4 Oct 2006 00:30:00 -0000 1.249
--- src/backend/utils/cache/relcache.c 27 Oct 2006 20:36:24 -0000
***************
*** 3568,3575 ****
if (beforeSend)
{
/* no interlock needed here */
! unlink(initfilename);
}
else
{
--- 3568,3599 ----
if (beforeSend)
{
+ /*
+ * Make a non-transactional XLOG entry showing the file removal
+ * It's non-transactional because we should replay it whether the
+ * transaction commits or not; the underlying file change is certainly
+ * not reversible. We do this *once* because we don't need to process
+ * invalidation messages during WAL reply, so even though the file
+ * is unlinked twice during normal invalidation sequence we need only
+ * do this once during WAL replay and so need only write WAL once.
+ */
+ XLogRecPtr lsn;
+ XLogRecData rdata;
+ xl_cache_init_file_inval xlrec;
+
+ xlrec.dbOid = MyDatabaseId;
+ xlrec.dbTableSpaceOid = MyDatabaseTableSpace;
+
+ rdata.data = (char *) &xlrec;
+ rdata.len = sizeof(xlrec);
+ rdata.buffer = InvalidBuffer;
+ rdata.next = NULL;
+
+ lsn = XLogInsert(RM_CACHE_ID, XLOG_CACHE_INIT_FILE_INVAL | XLOG_NO_TRAN,
+ &rdata);
+
/* no interlock needed here */
! unlink(initfilename);
}
else
{
***************
*** 3582,3588 ****
* that we will execute second and successfully unlink the file.
*/
LWLockAcquire(RelCacheInitLock, LW_EXCLUSIVE);
! unlink(initfilename);
LWLockRelease(RelCacheInitLock);
}
}
--- 3606,3651 ----
* that we will execute second and successfully unlink the file.
*/
LWLockAcquire(RelCacheInitLock, LW_EXCLUSIVE);
! unlink(initfilename);
LWLockRelease(RelCacheInitLock);
}
+
+ }
+
+ void
+ cache_redo(XLogRecPtr lsn, XLogRecord *record)
+ {
+ uint8 info = record->xl_info & ~XLR_INFO_MASK;
+
+ if (info == XLOG_CACHE_INIT_FILE_INVAL)
+ {
+ xl_cache_init_file_inval *xlrec = (xl_cache_init_file_inval *) XLogRecGetData(record);
+ char *dbpath = GetDatabasePath((int) &xlrec->dbOid, (int) &xlrec->dbTableSpaceOid);
+ char initfilename[MAXPGPATH];
+
+ snprintf(initfilename, sizeof(initfilename), "%s/%s",
+ dbpath, RELCACHE_INIT_FILENAME);
+
+ unlink(initfilename);
+ pfree(dbpath);
+ }
+ else
+ elog(PANIC, "cache_redo: unknown op code %u", info);
+ }
+
+ void
+ cache_desc(StringInfo buf, uint8 xl_info, char *rec)
+ {
+ uint8 info = xl_info & ~XLR_INFO_MASK;
+
+ if (info == XLOG_CACHE_INIT_FILE_INVAL)
+ {
+ xl_cache_init_file_inval *xlrec = (xl_cache_init_file_inval *) rec;
+
+ appendStringInfo(buf, "cache: relcache init file invalidation; ");
+ appendStringInfo(buf, "database %u; tablespace %u",
+ (int) &xlrec->dbOid, (int) &xlrec->dbTableSpaceOid);
+ }
+ else
+ appendStringInfo(buf, "UNKNOWN");
}
Index: src/include/access/rmgr.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/access/rmgr.h,v
retrieving revision 1.16
diff -c -r1.16 rmgr.h
*** src/include/access/rmgr.h 2 May 2006 11:28:55 -0000 1.16
--- src/include/access/rmgr.h 27 Oct 2006 20:36:25 -0000
***************
*** 20,25 ****
--- 20,26 ----
#define RM_DBASE_ID 4
#define RM_TBLSPC_ID 5
#define RM_MULTIXACT_ID 6
+ #define RM_CACHE_ID 7
#define RM_HEAP_ID 10
#define RM_BTREE_ID 11
#define RM_HASH_ID 12
Index: src/include/utils/relcache.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/utils/relcache.h,v
retrieving revision 1.55
diff -c -r1.55 relcache.h
*** src/include/utils/relcache.h 31 Jul 2006 20:09:10 -0000 1.55
--- src/include/utils/relcache.h 27 Oct 2006 20:36:26 -0000
***************
*** 64,69 ****
--- 64,84 ----
extern void AtEOSubXact_RelationCache(bool isCommit, SubTransactionId mySubid,
SubTransactionId parentSubid);
+ /*
+ * xlog support for cache invalidation
+ */
+ extern void cache_redo(XLogRecPtr lsn, XLogRecord *record);
+ extern void cache_desc(StringInfo buf, uint8 xl_info, char *rec);
+
+ typedef struct xl_cache_init_file_inval
+ {
+ Oid dbOid;
+ Oid dbTableSpaceOid;
+ } xl_cache_init_file_inval;
+
+ #define XLOG_CACHE_INIT_FILE_INVAL 0x00
+
+
/*
* Routines to help manage rebuilding of relcache init file
*/
"Simon Riggs" <simon@2ndquadrant.com> writes:
Enclose a patch for new WAL records for relcache invalidation.
I don't think this works. RelationCacheInitFileInvalidate is executed
post-commit, which means that there's a window between commit and where
you propose to write the WAL entry. A crash and restart in that
interval would leave the catalog changes committed, but not reflected
into pg_internal.init.
I think we're probably better off to just forcibly remove the init file
during post-recovery cleanup. The easiest place to do this might be
BuildFlatFiles, which has to scan pg_database anyway ...
regards, tom lane
On Wed, 2006-11-01 at 12:05 -0500, Tom Lane wrote:
I think we're probably better off to just forcibly remove the init file
during post-recovery cleanup. The easiest place to do this might be
BuildFlatFiles, which has to scan pg_database anyway ...
Presumably not rebuilding it, since we can let that occur naturally.
I'll submit a patch tomorrow.
--
Simon Riggs
EnterpriseDB http://www.enterprisedb.com
On Wed, 2006-11-01 at 12:05 -0500, Tom Lane wrote:
"Simon Riggs" <simon@2ndquadrant.com> writes:
Enclose a patch for new WAL records for relcache invalidation.
I don't think this works. RelationCacheInitFileInvalidate is executed
post-commit, which means that there's a window between commit and where
you propose to write the WAL entry. A crash and restart in that
interval would leave the catalog changes committed, but not reflected
into pg_internal.init.
Surely you are pointing out a bug, no?
If a backend did crash, the init file would be wrong and we'd get
exactly the same wrong relfilenode errors we got after that PITR.
The issue must surely be that the patch isn't wrong per se, just that
RelationCacheInitFileInvalidate is called too late and that requires an
additional fix. Are we certain that a crash between commit and
invalidation will cause a PANIC that takes down the server? Doesn't look
like its in a critical section to me.
I think we're probably better off to just forcibly remove the init file
during post-recovery cleanup. The easiest place to do this might be
BuildFlatFiles, which has to scan pg_database anyway ...
I can do this - I don't have a problem there, but the above issue just
occurred to me so I wonder now if its the right thing to do.
PITR will be always-safe but normal operation might not be.
--
Simon Riggs
EnterpriseDB http://www.enterprisedb.com
Tom, Simon et al; Please clarify.
PostgreSQL 8.1.5 on sparc-sun-solaris2.9, compiled by GCC gcc (GCC) 3.4.2
We're getting ready to init a new warm standby instance based on last
night's snapshot of running prod server. I see a few of these
pg_internal.init files in the cluster as it's being unpacked.
Same warm standby instance may sit for weeks gobbling up WALs from the
prod server then be finally brought live.
Question;
Is it safe to delete the .init files now (before starting recovery),
or perhaps unconditionally right after going live?
In other words, is there any sure fire preventative measure that I can
apply to guarantee against the fault that started this threadd?
Tom wrote:
Meanwhile, if you're trying to recover from a PITR backup and it's not
working, try removing any pg_internal.init files you can find.
Comment above seems to suggest not touching existing pg_internal.init
files unless a problem is seen.
Thanks
"Simon Riggs" <simon@2ndquadrant.com> writes:
On Wed, 2006-11-01 at 12:05 -0500, Tom Lane wrote:
"Simon Riggs" <simon@2ndquadrant.com> writes:
Enclose a patch for new WAL records for relcache invalidation.
I don't think this works. RelationCacheInitFileInvalidate is executed
post-commit, which means that there's a window between commit and where
you propose to write the WAL entry. A crash and restart in that
interval would leave the catalog changes committed, but not reflected
into pg_internal.init.Surely you are pointing out a bug, no?
If a backend did crash, the init file would be wrong and we'd get
exactly the same wrong relfilenode errors we got after that PITR.The issue must surely be that the patch isn't wrong per se, just that
RelationCacheInitFileInvalidate is called too late and that requires an
additional fix. Are we certain that a crash between commit and
invalidation will cause a PANIC that takes down the server? Doesn't look
like its in a critical section to me.I think we're probably better off to just forcibly remove the init file
during post-recovery cleanup. The easiest place to do this might be
BuildFlatFiles, which has to scan pg_database anyway ...I can do this - I don't have a problem there, but the above issue just
occurred to me so I wonder now if its the right thing to do.PITR will be always-safe but normal operation might not be.
--
Simon Riggs
EnterpriseDB http://www.enterprisedb.com---------------------------(end of broadcast)---------------------------
TIP 1: if posting/reading through Usenet, please send an appropriate
subscribe-nomail command to majordomo@postgresql.org so that your
message can get through to the mailing list cleanly
--
-------------------------------------------------------------------------------
Jerry Sievers 305 854-3001 (home) WWW ECommerce Consultant
305 321-1144 (mobile http://www.JerrySievers.com/
On Thu, 2006-11-02 at 09:36 -0500, Jerry Sievers wrote:
Is it safe to delete the .init files now (before starting recovery),
or perhaps unconditionally right after going live?
You're safe to delete those files from the restored version of your base
backup prior to commencing startup with a recovery.conf.
When they are missing they will be recreated automatically.
When we agree on how to fix it, which should be soon, I would imagine
that the fix can be back patched to 8.0 and 8.1 and fixed prior to
release in 8.2.
--
Simon Riggs
EnterpriseDB http://www.enterprisedb.com
"Simon Riggs" <simon@2ndquadrant.com> writes:
On Wed, 2006-11-01 at 12:05 -0500, Tom Lane wrote:
I don't think this works.
Surely you are pointing out a bug, no?
If a backend did crash, the init file would be wrong and we'd get
exactly the same wrong relfilenode errors we got after that PITR.
Yeah, which is the same bug we've got now. They're both WAL-replay-
doesn't-fix-the-init-file cases.
The issue must surely be that the patch isn't wrong per se, just that
RelationCacheInitFileInvalidate is called too late and that requires an
additional fix.
No. In the non-crash situation there is sufficient interlocking to
avoid a problem, and I feel no desire to redesign that mechanism.
Trying to do it before commit would create its own issues, anyway:
someone could install a new init file before you finish committing.
regards, tom lane
Jerry Sievers <jerry@jerrysievers.com> writes:
Is it safe to delete the .init files now (before starting recovery),
Should be OK as far as I can see.
regards, tom lane
On Wed, 2006-11-01 at 12:05 -0500, Tom Lane wrote:
I think we're probably better off to just forcibly remove the init file
during post-recovery cleanup. The easiest place to do this might be
BuildFlatFiles, which has to scan pg_database anyway ...
Patch enclosed.
Clean apply to HEAD, make check OK, plus restart check.
No specific PITR test, since same code path.
--
Simon Riggs
EnterpriseDB http://www.enterprisedb.com
Attachments:
unlink_pg_internal_at_startup.patchtext/x-patch; charset=UTF-8; name=unlink_pg_internal_at_startup.patchDownload
Index: src/backend/utils/cache/inval.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/utils/cache/inval.c,v
retrieving revision 1.78
diff -c -r1.78 inval.c
*** src/backend/utils/cache/inval.c 4 Oct 2006 00:30:00 -0000 1.78
--- src/backend/utils/cache/inval.c 5 Nov 2006 22:00:03 -0000
***************
*** 767,776 ****
SendSharedInvalidMessage(msg);
break;
case TWOPHASE_INFO_FILE_BEFORE:
! RelationCacheInitFileInvalidate(true);
break;
case TWOPHASE_INFO_FILE_AFTER:
! RelationCacheInitFileInvalidate(false);
break;
default:
Assert(false);
--- 767,776 ----
SendSharedInvalidMessage(msg);
break;
case TWOPHASE_INFO_FILE_BEFORE:
! RelationCacheInitFileInvalidate(true, DatabasePath);
break;
case TWOPHASE_INFO_FILE_AFTER:
! RelationCacheInitFileInvalidate(false, DatabasePath);
break;
default:
Assert(false);
***************
*** 817,823 ****
* unless we committed.
*/
if (transInvalInfo->RelcacheInitFileInval)
! RelationCacheInitFileInvalidate(true);
AppendInvalidationMessages(&transInvalInfo->PriorCmdInvalidMsgs,
&transInvalInfo->CurrentCmdInvalidMsgs);
--- 817,823 ----
* unless we committed.
*/
if (transInvalInfo->RelcacheInitFileInval)
! RelationCacheInitFileInvalidate(true, DatabasePath);
AppendInvalidationMessages(&transInvalInfo->PriorCmdInvalidMsgs,
&transInvalInfo->CurrentCmdInvalidMsgs);
***************
*** 826,832 ****
SendSharedInvalidMessage);
if (transInvalInfo->RelcacheInitFileInval)
! RelationCacheInitFileInvalidate(false);
}
else if (transInvalInfo != NULL)
{
--- 826,832 ----
SendSharedInvalidMessage);
if (transInvalInfo->RelcacheInitFileInval)
! RelationCacheInitFileInvalidate(false, DatabasePath);
}
else if (transInvalInfo != NULL)
{
Index: src/backend/utils/cache/relcache.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/utils/cache/relcache.c,v
retrieving revision 1.249
diff -c -r1.249 relcache.c
*** src/backend/utils/cache/relcache.c 4 Oct 2006 00:30:00 -0000 1.249
--- src/backend/utils/cache/relcache.c 5 Nov 2006 22:00:06 -0000
***************
*** 3559,3570 ****
* no backend has been started since the last removal.
*/
void
! RelationCacheInitFileInvalidate(bool beforeSend)
{
char initfilename[MAXPGPATH];
snprintf(initfilename, sizeof(initfilename), "%s/%s",
! DatabasePath, RELCACHE_INIT_FILENAME);
if (beforeSend)
{
--- 3559,3570 ----
* no backend has been started since the last removal.
*/
void
! RelationCacheInitFileInvalidate(bool beforeSend, char *ForDatabasePath)
{
char initfilename[MAXPGPATH];
snprintf(initfilename, sizeof(initfilename), "%s/%s",
! ForDatabasePath, RELCACHE_INIT_FILENAME);
if (beforeSend)
{
Index: src/backend/utils/init/flatfiles.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/utils/init/flatfiles.c,v
retrieving revision 1.21
diff -c -r1.21 flatfiles.c
*** src/backend/utils/init/flatfiles.c 14 Jul 2006 14:52:25 -0000 1.21
--- src/backend/utils/init/flatfiles.c 5 Nov 2006 22:00:07 -0000
***************
*** 36,41 ****
--- 36,42 ----
#include "access/transam.h"
#include "access/twophase_rmgr.h"
#include "access/xact.h"
+ #include "catalog/catalog.h"
#include "catalog/pg_auth_members.h"
#include "catalog/pg_authid.h"
#include "catalog/pg_database.h"
***************
*** 47,52 ****
--- 48,54 ----
#include "storage/pmsignal.h"
#include "utils/builtins.h"
#include "utils/flatfiles.h"
+ #include "utils/relcache.h"
#include "utils/resowner.h"
***************
*** 165,173 ****
*
* A side effect is to determine the oldest database's datminxid
* so we can set or update the XID wrap limit.
*/
static void
! write_database_file(Relation drel)
{
char *filename,
*tempname;
--- 167,178 ----
*
* A side effect is to determine the oldest database's datminxid
* so we can set or update the XID wrap limit.
+ *
+ * removeDBRelInitFile is true only at startup, to allow us to
+ * remove the relcache init file as we read each database
*/
static void
! write_database_file(Relation drel, bool removeDBRelInitFile)
{
char *filename,
*tempname;
***************
*** 252,257 ****
--- 257,278 ----
fputs_quote(datname, fp);
fprintf(fp, " %u %u %u %u\n",
datoid, dattablespace, datminxid, datvacuumxid);
+
+ /*
+ * We call this at startup to remove the relcache init file
+ */
+ if (removeDBRelInitFile)
+ {
+ char *dbpath = GetDatabasePath(datoid, dattablespace);
+
+ ereport(DEBUG3,
+ (errmsg("removing pg_internal.init for database \%s\ ",
+ dbpath)));
+
+ /* We call this once only, without interlock */
+ RelationCacheInitFileInvalidate(true, dbpath);
+ pfree(dbpath);
+ }
}
heap_endscan(scan);
***************
*** 703,709 ****
/* No locking is needed because no one else is alive yet */
rel_db = XLogOpenRelation(rnode);
! write_database_file(rel_db);
if (!database_only)
{
--- 724,734 ----
/* No locking is needed because no one else is alive yet */
rel_db = XLogOpenRelation(rnode);
!
! /*
! * Write database_file, plus unlink any rel cache init files
! */
! write_database_file(rel_db, true);
if (!database_only)
{
***************
*** 815,821 ****
if (database_file_update_subid != InvalidSubTransactionId)
{
database_file_update_subid = InvalidSubTransactionId;
! write_database_file(drel);
heap_close(drel, NoLock);
}
--- 840,846 ----
if (database_file_update_subid != InvalidSubTransactionId)
{
database_file_update_subid = InvalidSubTransactionId;
! write_database_file(drel, false);
heap_close(drel, NoLock);
}
Index: src/include/utils/relcache.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/utils/relcache.h,v
retrieving revision 1.55
diff -c -r1.55 relcache.h
*** src/include/utils/relcache.h 31 Jul 2006 20:09:10 -0000 1.55
--- src/include/utils/relcache.h 5 Nov 2006 22:00:09 -0000
***************
*** 68,74 ****
* Routines to help manage rebuilding of relcache init file
*/
extern bool RelationIdIsInInitFile(Oid relationId);
! extern void RelationCacheInitFileInvalidate(bool beforeSend);
/* should be used only by relcache.c and catcache.c */
extern bool criticalRelcachesBuilt;
--- 68,74 ----
* Routines to help manage rebuilding of relcache init file
*/
extern bool RelationIdIsInInitFile(Oid relationId);
! extern void RelationCacheInitFileInvalidate(bool beforeSend, char *ForDatabasePath);
/* should be used only by relcache.c and catcache.c */
extern bool criticalRelcachesBuilt;
"Simon Riggs" <simon@2ndquadrant.com> writes:
On Wed, 2006-11-01 at 12:05 -0500, Tom Lane wrote:
I think we're probably better off to just forcibly remove the init file
during post-recovery cleanup. The easiest place to do this might be
BuildFlatFiles, which has to scan pg_database anyway ...
Patch enclosed.
Applied to HEAD and 8.1. It doesn't work in 8.0 though, because
flatfiles.c doesn't exist. AFAIR there isn't any startup-time scan
of pg_database at all before 8.1. Not sure if it's worth coming up
with a whole new patch for that branch.
regards, tom lane