? Makefile.global ? pitr_arch_v3.01.patch ? pitr_v3.1.patch ? backend/postgres ? backend/catalog/postgres.bki ? backend/catalog/postgres.description ? backend/postmaster/pgarch.c ? backend/postmaster/postmaster.c.new ? backend/utils/mb/conversion_procs/conversion_create.sql ? backend/utils/mb/conversion_procs/ascii_and_mic/libascii_and_mic.so.0.0 ? backend/utils/mb/conversion_procs/cyrillic_and_mic/libcyrillic_and_mic.so.0.0 ? backend/utils/mb/conversion_procs/euc_cn_and_mic/libeuc_cn_and_mic.so.0.0 ? backend/utils/mb/conversion_procs/euc_jp_and_sjis/libeuc_jp_and_sjis.so.0.0 ? backend/utils/mb/conversion_procs/euc_kr_and_mic/libeuc_kr_and_mic.so.0.0 ? backend/utils/mb/conversion_procs/euc_tw_and_big5/libeuc_tw_and_big5.so.0.0 ? backend/utils/mb/conversion_procs/latin2_and_win1250/liblatin2_and_win1250.so.0.0 ? backend/utils/mb/conversion_procs/latin_and_mic/liblatin_and_mic.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_ascii/libutf8_and_ascii.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_big5/libutf8_and_big5.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_cyrillic/libutf8_and_cyrillic.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_euc_cn/libutf8_and_euc_cn.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_euc_jp/libutf8_and_euc_jp.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_euc_kr/libutf8_and_euc_kr.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_euc_tw/libutf8_and_euc_tw.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_gb18030/libutf8_and_gb18030.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_gbk/libutf8_and_gbk.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_iso8859/libutf8_and_iso8859.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_iso8859_1/libutf8_and_iso8859_1.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_johab/libutf8_and_johab.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_sjis/libutf8_and_sjis.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_tcvn/libutf8_and_tcvn.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_uhc/libutf8_and_uhc.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_win1250/libutf8_and_win1250.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_win1256/libutf8_and_win1256.so.0.0 ? backend/utils/mb/conversion_procs/utf8_and_win874/libutf8_and_win874.so.0.0 ? bin/initdb/initdb ? bin/initlocation/initlocation ? bin/ipcclean/ipcclean ? bin/pg_config/pg_config ? bin/pg_controldata/pg_controldata ? bin/pg_ctl/pg_ctl ? bin/pg_dump/pg_dump ? bin/pg_dump/pg_dumpall ? bin/pg_dump/pg_restore ? bin/pg_resetxlog/pg_resetxlog ? bin/psql/psql ? bin/scripts/clusterdb ? bin/scripts/createdb ? bin/scripts/createlang ? bin/scripts/createuser ? bin/scripts/dropdb ? bin/scripts/droplang ? bin/scripts/dropuser ? bin/scripts/vacuumdb ? include/pg_config.h ? include/pgarch.h ? include/stamp-h ? interfaces/ecpg/compatlib/libecpg_compat.so.1.1 ? interfaces/ecpg/ecpglib/libecpg.so.4.2 ? interfaces/ecpg/pgtypeslib/libpgtypes.so.1.2 ? interfaces/ecpg/preproc/ecpg ? interfaces/libpq/libpq.so.3.2 ? pl/plpgsql/src/libplpgsql.so.1.0 ? port/pg_config_paths.h ? timezone/zic Index: backend/access/nbtree/nbtsort.c =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/backend/access/nbtree/nbtsort.c,v retrieving revision 1.82 diff -c -r1.82 nbtsort.c *** backend/access/nbtree/nbtsort.c 2 Jun 2004 17:28:17 -0000 1.82 --- backend/access/nbtree/nbtsort.c 15 Jun 2004 15:19:48 -0000 *************** *** 67,72 **** --- 67,73 ---- #include "miscadmin.h" #include "storage/smgr.h" #include "utils/tuplesort.h" + #include "access/xlog.h" /* *************** *** 220,235 **** wstate.index = btspool->index; /* ! * We need to log index creation in WAL iff WAL archiving is enabled * AND it's not a temp index. - * - * XXX when WAL archiving is actually supported, this test will likely - * need to change; and the hardwired extern is cruddy anyway ... */ { ! extern char XLOG_archive_dir[]; ! ! wstate.btws_use_wal = XLOG_archive_dir[0] && !wstate.index->rd_istemp; } /* reserve the metapage */ wstate.btws_pages_alloced = BTREE_METAPAGE + 1; --- 221,231 ---- wstate.index = btspool->index; /* ! * We need to log index creation in WAL if WAL archiving is enabled * AND it's not a temp index. */ { ! wstate.btws_use_wal = XLogArchiveMode && !wstate.index->rd_istemp; } /* reserve the metapage */ wstate.btws_pages_alloced = BTREE_METAPAGE + 1; Index: backend/access/transam/xlog.c =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/backend/access/transam/xlog.c,v retrieving revision 1.146 diff -c -r1.146 xlog.c *** backend/access/transam/xlog.c 3 Jun 2004 02:08:00 -0000 1.146 --- backend/access/transam/xlog.c 15 Jun 2004 15:19:52 -0000 *************** *** 35,40 **** --- 35,41 ---- #include "storage/proc.h" #include "storage/sinval.h" #include "storage/spin.h" + #include "storage/pmsignal.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/relcache.h" *************** *** 84,95 **** /* User-settable parameters */ int CheckPointSegments = 3; int XLOGbuffers = 8; char *XLOG_sync_method = NULL; const char XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR; - char XLOG_archive_dir[MAXPGPATH]; /* null string means - * delete 'em */ #ifdef WAL_DEBUG bool XLOG_DEBUG = false; --- 85,98 ---- /* User-settable parameters */ + bool XLogArchiveMode = false; + bool XLogArchiveDEBUG = false; + char *XLogArchiveDest; + char *XLogArchiveProgram; int CheckPointSegments = 3; int XLOGbuffers = 8; char *XLOG_sync_method = NULL; const char XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR; #ifdef WAL_DEBUG bool XLOG_DEBUG = false; *************** *** 392,397 **** --- 395,401 ---- /* File path names */ static char XLogDir[MAXPGPATH]; + static char RLogDir[MAXPGPATH]; static char ControlFilePath[MAXPGPATH]; /* *************** *** 433,438 **** --- 437,444 ---- static bool InRedo = false; + static bool XLogArchiveNotify(uint32 log, uint32 seg); + static bool XLogArchiveDone(char xlog[MAXPGPATH]); static bool AdvanceXLInsertBuffer(void); static bool WasteXLInsertBuffer(void); *************** *** 911,916 **** --- 917,1041 ---- } /* + * XLogArchive API calls + * + * Two calls implement the backend side of the XLogArchive API + * XLogArchiveNotify + * XLogArchiveDone + */ + + /* + * XLogArchiveNotify + * + * Writes an archive notification file to the RLogDir + * + * The name of the notification file is the message that will be picked up + * by the archiver, e.g. we write RLogDir/00000001000000C6.ready + * and the archiver then knows to archive XLogDir/00000001000000C6, + * then when complete, rename it to RLogDir/00000001000000C6.done + * + * Called only when in XLogArchiveMode by one backend process + */ + static bool + XLogArchiveNotify(uint32 log, uint32 seg) + { + char rlog[32]; + char rlogpath[MAXPGPATH]; + FILE *rlogFD; + + /* insert an otherwise empty file called .ready */ + sprintf(rlog, "%08X%08X.ready", log, seg); + snprintf(rlogpath, MAXPGPATH, "%s/%s", RLogDir, rlog); + rlogFD = AllocateFile(rlogpath, "w"); + if (rlogFD == NULL) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write archive_status file \"%s\" ", + rlogpath))); + FreeFile(rlogFD); + + /* the existence of this file is the message to the archiver to identify + * which files require archiving + * + * if this file is written OK, we then signal the ARCHIVER to do its thang + */ + + if (XLogArchiveDEBUG) + elog(LOG, "backend: written %s", rlogpath ); + + SendPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER); + + return true; + } + + /* + * XLogArchiveDone + * + * Searches for an archive notification file in RLogDir + * + * Reads RLogDir looking for a specific filename. If that filename ends with .done + * then we know that the filename refers to an xlog in XLogDir that is safe to + * recycle. If the filename ends .ready then thats OK, else we have an error. + * + * Called only when in XLogArchiveMode by bgwriter (when performing checkpoint) + * + */ + static bool + XLogArchiveDone(char xlog[32]) + { + char rlogpath[MAXPGPATH]; + FILE *rlogFD; + + if (XLogArchiveDEBUG) + elog(LOG, "chkpt: checking for log file %s", + xlog); + + /* If .done exists then return XLA_ARCHIVE_OK */ + snprintf(rlogpath, MAXPGPATH, "%s/%s.done", RLogDir, xlog); + rlogFD = AllocateFile(rlogpath, "r"); + if (rlogFD != NULL) { + FreeFile(rlogFD); + if (XLogArchiveDEBUG) + elog(LOG, "chkpt: archiving done for log file %s", + xlog); + + /* really ought to wait until after xlog recycled, in case there + * is an error between the two activities -- do that later + */ + unlink(rlogpath); + return true; + } + else + { + /* + * else if .ready exists then return false and issue WARNING + * ...this indicates archiver is either not working at all or + * if it is, then its just way too slow or incorrectly configured + */ + snprintf(rlogpath, MAXPGPATH, "%s/%s.ready", RLogDir, xlog); + rlogFD = AllocateFile(rlogpath, "r"); + if (rlogFD != NULL) { + FreeFile(rlogFD); + elog(WARNING, "chkpt: archiving not yet started for log file %s", + xlog); + return false; + } + else + { + /* else issue a WARNING.... a notification file SHOULD exist...unless the + * database has just been restored in which case it may be absent, so + * issue a WARNING, not an error, then return + */ + ereport(WARNING, + (errcode_for_file_access(), + errmsg("chkpt: cannot find archive_status file: %s ", + rlogpath))); + return false; + } + } + } + + /* * Advance the Insert state to the next buffer page, writing out the next * buffer if it still contains unwritten data. * *************** *** 1259,1264 **** --- 1384,1397 ---- { issue_xlog_fsync(); LogwrtResult.Flush = LogwrtResult.Write; /* end of current page */ + + /* + * Notify xlog ready to archive? + */ + if (XLogArchiveMode && !XLogArchiveNotify(openLogId, openLogSeg)) + elog(WARNING, "could not set notify for archiver to read log file %u, segment %u", + openLogId, openLogSeg); + } if (ispartialpage) *************** *** 1761,1785 **** errno = 0; while ((xlde = readdir(xldir)) != NULL) { if (strlen(xlde->d_name) == 16 && strspn(xlde->d_name, "0123456789ABCDEF") == 16 && strcmp(xlde->d_name, lastoff) <= 0) { snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name); ! if (XLOG_archive_dir[0]) ! { ! ereport(LOG, ! (errmsg("archiving transaction log file \"%s\"", ! xlde->d_name))); ! elog(WARNING, "archiving log files is not implemented"); ! } ! else { /* * Before deleting the file, see if it can be recycled as * a future log segment. We allow recycling segments up ! * to XLOGfileslop segments beyond the current XLOG ! * location. */ if (InstallXLogFileSegment(endlogId, endlogSeg, path, true, XLOGfileslop, --- 1894,1919 ---- errno = 0; while ((xlde = readdir(xldir)) != NULL) { + /* if correct length and alphanumeric makeup of file looks correct + * use the alphanumeric sorting property of the filenames to decide + * which ones are earlier than the lastoff transaction log + * ...maybe should read lastwrite datetime of lastoff, then check that + * only files last written earlier than this are removed/recycled + */ if (strlen(xlde->d_name) == 16 && strspn(xlde->d_name, "0123456789ABCDEF") == 16 && strcmp(xlde->d_name, lastoff) <= 0) { snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name); ! if ( !XLogArchiveMode || ! (XLogArchiveMode && XLogArchiveDone(xlde->d_name)) ! ) { /* * Before deleting the file, see if it can be recycled as * a future log segment. We allow recycling segments up ! * until there are XLOGfileslop segments beyond the ! * current XLOG location, otherwise they are removed. */ if (InstallXLogFileSegment(endlogId, endlogSeg, path, true, XLOGfileslop, *************** *** 1793,1799 **** { /* No need for any more future segments... */ ereport(LOG, ! (errmsg("removing transaction log file \"%s\"", xlde->d_name))); unlink(path); } --- 1927,1933 ---- { /* No need for any more future segments... */ ereport(LOG, ! (errmsg("too many transaction log files, removing \"%s\"", xlde->d_name))); unlink(path); } *************** *** 2254,2259 **** --- 2388,2394 ---- { /* Init XLOG file paths */ snprintf(XLogDir, MAXPGPATH, "%s/pg_xlog", DataDir); + snprintf(RLogDir, MAXPGPATH, "%s/archive_status", XLogDir); snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir); } Index: backend/postmaster/Makefile =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/backend/postmaster/Makefile,v retrieving revision 1.15 diff -c -r1.15 Makefile *** backend/postmaster/Makefile 29 May 2004 22:48:19 -0000 1.15 --- backend/postmaster/Makefile 15 Jun 2004 15:19:53 -0000 *************** *** 12,18 **** top_builddir = ../../.. include $(top_builddir)/src/Makefile.global ! OBJS = postmaster.o bgwriter.o pgstat.o all: SUBSYS.o --- 12,18 ---- top_builddir = ../../.. include $(top_builddir)/src/Makefile.global ! OBJS = postmaster.o bgwriter.o pgstat.o pgarch.o all: SUBSYS.o Index: backend/postmaster/postmaster.c =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/backend/postmaster/postmaster.c,v retrieving revision 1.404 diff -c -r1.404 postmaster.c *** backend/postmaster/postmaster.c 14 Jun 2004 18:08:19 -0000 1.404 --- backend/postmaster/postmaster.c 15 Jun 2004 15:19:58 -0000 *************** *** 111,117 **** #include "utils/ps_status.h" #include "bootstrap/bootstrap.h" #include "pgstat.h" ! /* * List of active backends (or child processes anyway; we don't actually --- 111,117 ---- #include "utils/ps_status.h" #include "bootstrap/bootstrap.h" #include "pgstat.h" ! #include "pgarch.h" /* * List of active backends (or child processes anyway; we don't actually *************** *** 812,817 **** --- 812,824 ---- */ whereToSendOutput = None; + /* + * Initialize and try to startup the archiver process + */ + if (XLogArchiveMode) { + pgarch_start(); + } + /* * Initialize the statistics collector stuff */ *************** *** 1148,1153 **** --- 1155,1164 ---- kill(BgWriterPID, SIGUSR2); } + /* If we have lost the archiver, try to start a new one */ + if (Shutdown == NoShutdown && XLogArchiveMode && !pgarch_is_running) + pgarch_start(); + /* If we have lost the stats collector, try to start a new one */ if (PgStatPID == 0 && StartupPID == 0 && !FatalError && Shutdown == NoShutdown) *************** *** 1869,1874 **** --- 1880,1898 ---- #endif /* WIN32 */ #endif /* HAVE_WAITPID */ + /* + * Check if this child was the archiver. If so, try to + * start a new one. (If fail, we'll try again in future cycles of + * the main loop.) + */ + if (XLogArchiveMode && pgarch_ispgarch(pid)) + { + LogChildExit(LOG, gettext("archiver process"), + pid, exitstatus); + pgarch_start(); + continue; + } + /* * Check if this child was a startup process. */ *************** *** 2882,2887 **** --- 2906,2925 ---- if (Shutdown <= SmartShutdown) SignalChildren(SIGUSR1); } + + if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER)) + { + /* + * Send SIGUSR1 to ARCHIVER process, to wake it up and begin + * archiving next transaction log file. Backend should only + * send if in XLogArchiveMode... + */ + if (XLogArchiveMode && Shutdown == NoShutdown) { + if (XLogArchiveDEBUG) + elog(LOG, "postmaster: WAKEN_ARCHIVER received, sending SIGUSR1 to archiver"); + SendArchiverSignal(); + } + } PG_SETMASK(&UnBlockSig); Index: backend/utils/misc/guc.c =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/backend/utils/misc/guc.c,v retrieving revision 1.211 diff -c -r1.211 guc.c *** backend/utils/misc/guc.c 11 Jun 2004 03:54:54 -0000 1.211 --- backend/utils/misc/guc.c 15 Jun 2004 15:20:16 -0000 *************** *** 371,376 **** --- 371,392 ---- static struct config_bool ConfigureNamesBool[] = { + { + {"archive_mode", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Enable archiving of full transaction log files to a specified archival destination."), + NULL + }, + &XLogArchiveMode, + false, NULL, NULL + }, + { + {"archive_debug", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Provide debug output for archive activities."), + NULL + }, + &XLogArchiveDEBUG, + false, NULL, NULL + }, { {"enable_seqscan", PGC_USERSET, QUERY_TUNING_METHOD, gettext_noop("Enables the planner's use of sequential-scan plans."), *************** *** 1400,1405 **** --- 1416,1439 ---- static struct config_string ConfigureNamesString[] = { + { + {"archive_dest", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Specifies where to archive WAL logs."), + gettext_noop("A directory or specific location for archiving transation log files from PostgreSQL") + }, + &XLogArchiveDest, + "", NULL, NULL + }, + + { + {"archive_program", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Archive program"), + gettext_noop("The external program that will be called to execute the archival process") + }, + &XLogArchiveProgram, + "", NULL, NULL + }, + { {"client_encoding", PGC_USERSET, CLIENT_CONN_LOCALE, gettext_noop("Sets the client's character set encoding."), Index: backend/utils/misc/postgresql.conf.sample =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/backend/utils/misc/postgresql.conf.sample,v retrieving revision 1.113 diff -c -r1.113 postgresql.conf.sample *** backend/utils/misc/postgresql.conf.sample 7 Apr 2004 05:05:50 -0000 1.113 --- backend/utils/misc/postgresql.conf.sample 15 Jun 2004 15:20:16 -0000 *************** *** 103,108 **** --- 103,119 ---- #--------------------------------------------------------------------------- + # ARCHIVING + #--------------------------------------------------------------------------- + + # - Settings - + + #archive_mode = true # enables archiving of full txn log files + #archive_dest = '/tmp' # specifies destination of archive files + #archive_program = 'cp %s %s' # external archiving program command line + + + #--------------------------------------------------------------------------- # QUERY TUNING #--------------------------------------------------------------------------- Index: bin/initdb/initdb.c =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/bin/initdb/initdb.c,v retrieving revision 1.37 diff -c -r1.37 initdb.c *** bin/initdb/initdb.c 10 Jun 2004 22:26:20 -0000 1.37 --- bin/initdb/initdb.c 15 Jun 2004 15:20:17 -0000 *************** *** 1785,1791 **** char *pgdenv; /* PGDATA value got from sent to * environment */ char *subdirs[] = ! {"global", "pg_xlog", "pg_clog", "base", "base/1"}; progname = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "initdb"); --- 1785,1791 ---- char *pgdenv; /* PGDATA value got from sent to * environment */ char *subdirs[] = ! {"global", "pg_xlog", "pg_xlog/archive_status", "pg_clog", "base", "base/1"}; progname = get_progname(argv[0]); set_pglocale_pgservice(argv[0], "initdb"); Index: include/access/xlog.h =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/include/access/xlog.h,v retrieving revision 1.51 diff -c -r1.51 xlog.h *** include/access/xlog.h 29 May 2004 22:48:22 -0000 1.51 --- include/access/xlog.h 15 Jun 2004 15:20:17 -0000 *************** *** 210,215 **** --- 210,219 ---- extern int XLOGbuffers; extern char *XLOG_sync_method; extern const char XLOG_sync_method_default[]; + extern bool XLogArchiveMode; + extern bool XLogArchiveDEBUG; + extern char *XLogArchiveDest; + extern char *XLogArchiveProgram; #ifdef WAL_DEBUG extern bool XLOG_DEBUG; Index: include/storage/pmsignal.h =================================================================== RCS file: /projects/cvsroot/pgsql-server/src/include/storage/pmsignal.h,v retrieving revision 1.8 diff -c -r1.8 pmsignal.h *** include/storage/pmsignal.h 29 May 2004 22:48:23 -0000 1.8 --- include/storage/pmsignal.h 15 Jun 2004 15:20:18 -0000 *************** *** 24,29 **** --- 24,30 ---- { PMSIGNAL_PASSWORD_CHANGE, /* pg_pwd file has changed */ PMSIGNAL_WAKEN_CHILDREN, /* send a SIGUSR1 signal to all backends */ + PMSIGNAL_WAKEN_ARCHIVER, /* send a NOTIFY signal to ARCHIVER */ NUM_PMSIGNALS /* Must be last value of enum! */ } PMSignalReason;