From c5ca93c3bb759079c81aac3b507a5834864506c2 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 25 May 2018 09:43:16 +1200 Subject: [PATCH 4/6] Add tests for the undo log manager. Provide a test module that exercises undolog.c and undofile.c. TODO: A TAP test for recovery. Author: Thomas Munro Discussion: --- src/test/modules/Makefile | 1 + src/test/modules/test_undo/Makefile | 28 + .../modules/test_undo/expected/.gitignore | 1 + .../modules/test_undo/input/test_undo.source | 107 ++++ .../modules/test_undo/output/test_undo.source | 358 +++++++++++ src/test/modules/test_undo/sql/.gitignore | 1 + src/test/modules/test_undo/test_undo--1.0.sql | 53 ++ src/test/modules/test_undo/test_undo.c | 560 ++++++++++++++++++ src/test/modules/test_undo/test_undo.control | 4 + 9 files changed, 1113 insertions(+) create mode 100644 src/test/modules/test_undo/Makefile create mode 100644 src/test/modules/test_undo/expected/.gitignore create mode 100644 src/test/modules/test_undo/input/test_undo.source create mode 100644 src/test/modules/test_undo/output/test_undo.source create mode 100644 src/test/modules/test_undo/sql/.gitignore create mode 100644 src/test/modules/test_undo/test_undo--1.0.sql create mode 100644 src/test/modules/test_undo/test_undo.c create mode 100644 src/test/modules/test_undo/test_undo.control diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 19d60a506e1..43323a6f2ad 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_rbtree \ test_rls_hooks \ test_shm_mq \ + test_undo \ worker_spi $(recurse) diff --git a/src/test/modules/test_undo/Makefile b/src/test/modules/test_undo/Makefile new file mode 100644 index 00000000000..ce41746d0b8 --- /dev/null +++ b/src/test/modules/test_undo/Makefile @@ -0,0 +1,28 @@ +# src/test/modules/test_undo/Makefile + +MODULE_big = test_undo +OBJS = test_undo.o +PGFILEDESC = "test_undo - a test module for the undo log manager" + +EXTENSION = test_undo +DATA = test_undo--1.0.sql + +REGRESS = test_undo + +check: tablespace-setup + +.PHONY: tablespace-setup +tablespace-setup: + rm -fr testtablespace1 testtablespace2 + mkdir testtablespace1 testtablespace2 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_undo +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_undo/expected/.gitignore b/src/test/modules/test_undo/expected/.gitignore new file mode 100644 index 00000000000..1bb8bf6d7fd --- /dev/null +++ b/src/test/modules/test_undo/expected/.gitignore @@ -0,0 +1 @@ +# empty diff --git a/src/test/modules/test_undo/input/test_undo.source b/src/test/modules/test_undo/input/test_undo.source new file mode 100644 index 00000000000..21d1ee6e2bc --- /dev/null +++ b/src/test/modules/test_undo/input/test_undo.source @@ -0,0 +1,107 @@ +create extension test_undo; + +create view undo_logs as + select log_number, + persistence, + tablespace, + discard, + insert, + "end", + pid = pg_backend_pid() as my_pid, + xid = txid_current()::text::xid as my_xid + from pg_stat_undo_logs; + +begin; + +-- permanent storage +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +select * from undo_logs order by log_number; +-- write a short message +select undo_append('[permanent]'::bytea, 'permanent'); +select * from undo_logs order by log_number; +-- see if we can read it back +select undo_dump('000000000000003C', 11, 'permanent'); + +-- unlogged storage +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'unlogged'); +select * from undo_logs order by log_number; +-- write a short message +select undo_append(' '::bytea, 'unlogged'); +select * from undo_logs order by log_number; +-- see if we can read it back +select undo_dump('000001000000003C', 11, 'unlogged'); + +-- temporary storage +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'temporary'); +select * from undo_logs order by log_number; +-- write a short message +select undo_append('{temporary}'::bytea, 'temporary'); +select * from undo_logs order by log_number; +-- see if we can read it back +select undo_dump('000002000000003C', 11, 'temporary'); + +-- discard the data we wrote in each of those logs +select undo_discard('0000000000000047'); +select * from undo_logs order by log_number; +select undo_discard('0000010000000047'); +select * from undo_logs order by log_number; +select undo_discard('0000020000000047'); +select * from undo_logs order by log_number; + +commit; + +create tablespace ts1 location '@testtablespace@1'; +create tablespace ts2 location '@testtablespace@2'; + +begin; +set undo_tablespaces = ts1; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +select * from undo_logs order by log_number; +-- write a short message +select undo_append('ts1:perm---'::bytea, 'permanent'); +select * from undo_logs order by log_number; +-- discard +select undo_discard('0000030000000047'); +select * from undo_logs order by log_number; +commit; + +begin; +set undo_tablespaces = ts2; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +select * from undo_logs order by log_number; +-- write a short message +select undo_append('ts2:perm---', 'permanent'); +select * from undo_logs order by log_number; +-- discard +select undo_discard('0000040000000047'); +select * from undo_logs order by log_number; +commit; + +-- check that we can drop tablespaces (because there is nothing in them) +drop tablespace ts1; +drop tablespace ts2; + +-- we fail to allocate space now that ts2 is gone +begin; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +select * from undo_logs order by log_number; +commit; + +-- we go back to allocating from log 0 if we clear the GUC +begin; +set undo_tablespaces = ''; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +select * from undo_logs order by log_number; +-- discard +select undo_discard('000000000000006B'); +select * from undo_logs order by log_number; +commit; + +drop view undo_logs; diff --git a/src/test/modules/test_undo/output/test_undo.source b/src/test/modules/test_undo/output/test_undo.source new file mode 100644 index 00000000000..13ec679ae94 --- /dev/null +++ b/src/test/modules/test_undo/output/test_undo.source @@ -0,0 +1,358 @@ +create extension test_undo; +create view undo_logs as + select log_number, + persistence, + tablespace, + discard, + insert, + "end", + pid = pg_backend_pid() as my_pid, + xid = txid_current()::text::xid as my_xid + from pg_stat_undo_logs; +begin; +-- permanent storage +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +NOTICE: will copy 20 bytes into undo log at 0000000000000018 +NOTICE: writing chunk at offset 24 +NOTICE: will copy 16 bytes into undo log at 000000000000002C +NOTICE: writing chunk at offset 44 + undo_append_transaction_header +-------------------------------- + 0000000000000018 +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000018 | 000000000000003C | 0000000000100000 | t | t +(1 row) + +-- write a short message +select undo_append('[permanent]'::bytea, 'permanent'); +NOTICE: will copy 11 bytes into undo log at 000000000000003C +NOTICE: writing chunk at offset 60 + undo_append +------------------ + 000000000000003C +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000018 | 0000000000000047 | 0000000000100000 | t | t +(1 row) + +-- see if we can read it back +select undo_dump('000000000000003C', 11, 'permanent'); +NOTICE: 0000000000000038: 00 00 00 00 5b 70 65 72 ....[per +NOTICE: 0000000000000040: 6d 61 6e 65 6e 74 5d 00 manent]. + undo_dump +----------- + +(1 row) + +-- unlogged storage +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'unlogged'); +NOTICE: will copy 20 bytes into undo log at 0000010000000018 +NOTICE: writing chunk at offset 24 +NOTICE: will copy 16 bytes into undo log at 000001000000002C +NOTICE: writing chunk at offset 44 + undo_append_transaction_header +-------------------------------- + 0000010000000018 +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000018 | 0000000000000047 | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000018 | 000001000000003C | 0000010000100000 | t | t +(2 rows) + +-- write a short message +select undo_append(' '::bytea, 'unlogged'); +NOTICE: will copy 11 bytes into undo log at 000001000000003C +NOTICE: writing chunk at offset 60 + undo_append +------------------ + 000001000000003C +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000018 | 0000000000000047 | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000018 | 0000010000000047 | 0000010000100000 | t | t +(2 rows) + +-- see if we can read it back +select undo_dump('000001000000003C', 11, 'unlogged'); +NOTICE: 0000010000000038: 00 00 00 00 3c 75 6e 6c .... . + undo_dump +----------- + +(1 row) + +-- temporary storage +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'temporary'); +NOTICE: will copy 20 bytes into undo log at 0000020000000018 +NOTICE: writing chunk at offset 24 +NOTICE: will copy 16 bytes into undo log at 000002000000002C +NOTICE: writing chunk at offset 44 + undo_append_transaction_header +-------------------------------- + 0000020000000018 +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000018 | 0000000000000047 | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000018 | 0000010000000047 | 0000010000100000 | t | t + 2 | temporary | pg_default | 0000020000000018 | 000002000000003C | 0000020000100000 | t | t +(3 rows) + +-- write a short message +select undo_append('{temporary}'::bytea, 'temporary'); +NOTICE: will copy 11 bytes into undo log at 000002000000003C +NOTICE: writing chunk at offset 60 + undo_append +------------------ + 000002000000003C +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000018 | 0000000000000047 | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000018 | 0000010000000047 | 0000010000100000 | t | t + 2 | temporary | pg_default | 0000020000000018 | 0000020000000047 | 0000020000100000 | t | t +(3 rows) + +-- see if we can read it back +select undo_dump('000002000000003C', 11, 'temporary'); +NOTICE: 0000020000000038: 00 00 00 00 7b 74 65 6d ....{tem +NOTICE: 0000020000000040: 70 6f 72 61 72 79 7d 00 porary}. + undo_dump +----------- + +(1 row) + +-- discard the data we wrote in each of those logs +select undo_discard('0000000000000047'); + undo_discard +-------------- + +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000018 | 0000010000000047 | 0000010000100000 | t | t + 2 | temporary | pg_default | 0000020000000018 | 0000020000000047 | 0000020000100000 | t | t +(3 rows) + +select undo_discard('0000010000000047'); + undo_discard +-------------- + +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | t | t + 2 | temporary | pg_default | 0000020000000018 | 0000020000000047 | 0000020000100000 | t | t +(3 rows) + +select undo_discard('0000020000000047'); + undo_discard +-------------- + +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | t | t + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | t | t +(3 rows) + +commit; +create tablespace ts1 location '@testtablespace@1'; +create tablespace ts2 location '@testtablespace@2'; +begin; +set undo_tablespaces = ts1; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +NOTICE: will copy 20 bytes into undo log at 0000030000000018 +NOTICE: writing chunk at offset 24 +NOTICE: will copy 16 bytes into undo log at 000003000000002C +NOTICE: writing chunk at offset 44 + undo_append_transaction_header +-------------------------------- + 0000030000000018 +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | | + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | + 3 | permanent | ts1 | 0000030000000018 | 000003000000003C | 0000030000100000 | t | t +(4 rows) + +-- write a short message +select undo_append('ts1:perm---'::bytea, 'permanent'); +NOTICE: will copy 11 bytes into undo log at 000003000000003C +NOTICE: writing chunk at offset 60 + undo_append +------------------ + 000003000000003C +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | | + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | + 3 | permanent | ts1 | 0000030000000018 | 0000030000000047 | 0000030000100000 | t | t +(4 rows) + +-- discard +select undo_discard('0000030000000047'); + undo_discard +-------------- + +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | | + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | + 3 | permanent | ts1 | 0000030000000047 | 0000030000000047 | 0000030000100000 | t | t +(4 rows) + +commit; +begin; +set undo_tablespaces = ts2; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +NOTICE: will copy 20 bytes into undo log at 0000040000000018 +NOTICE: writing chunk at offset 24 +NOTICE: will copy 16 bytes into undo log at 000004000000002C +NOTICE: writing chunk at offset 44 + undo_append_transaction_header +-------------------------------- + 0000040000000018 +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | | + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | + 3 | permanent | ts1 | 0000030000000047 | 0000030000000047 | 0000030000100000 | | + 4 | permanent | ts2 | 0000040000000018 | 000004000000003C | 0000040000100000 | t | t +(5 rows) + +-- write a short message +select undo_append('ts2:perm---', 'permanent'); +NOTICE: will copy 11 bytes into undo log at 000004000000003C +NOTICE: writing chunk at offset 60 + undo_append +------------------ + 000004000000003C +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | | + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | + 3 | permanent | ts1 | 0000030000000047 | 0000030000000047 | 0000030000100000 | | + 4 | permanent | ts2 | 0000040000000018 | 0000040000000047 | 0000040000100000 | t | t +(5 rows) + +-- discard +select undo_discard('0000040000000047'); + undo_discard +-------------- + +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 0000000000000047 | 0000000000100000 | | + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | + 3 | permanent | ts1 | 0000030000000047 | 0000030000000047 | 0000030000100000 | | + 4 | permanent | ts2 | 0000040000000047 | 0000040000000047 | 0000040000100000 | t | t +(5 rows) + +commit; +-- check that we can drop tablespaces (because there is nothing in them) +drop tablespace ts1; +drop tablespace ts2; +-- we fail to allocate space now that ts2 is gone +begin; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +ERROR: tablespace "ts2" does not exist +HINT: Create the tablespace or set undo_tablespaces to a valid or empty list. +select * from undo_logs order by log_number; +ERROR: current transaction is aborted, commands ignored until end of transaction block +commit; +-- we go back to allocating from log 0 if we clear the GUC +begin; +set undo_tablespaces = ''; +-- write a transaction header to avoid upsetting undo workers +select undo_append_transaction_header(txid_current()::text::xid, 'permanent'); +NOTICE: will copy 20 bytes into undo log at 0000000000000047 +NOTICE: writing chunk at offset 71 +NOTICE: will copy 16 bytes into undo log at 000000000000005B +NOTICE: writing chunk at offset 91 + undo_append_transaction_header +-------------------------------- + 0000000000000047 +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 0000000000000047 | 000000000000006B | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | +(3 rows) + +-- discard +select undo_discard('000000000000006B'); + undo_discard +-------------- + +(1 row) + +select * from undo_logs order by log_number; + log_number | persistence | tablespace | discard | insert | end | my_pid | my_xid +------------+-------------+------------+------------------+------------------+------------------+--------+-------- + 0 | permanent | pg_default | 000000000000006B | 000000000000006B | 0000000000100000 | t | t + 1 | unlogged | pg_default | 0000010000000047 | 0000010000000047 | 0000010000100000 | | + 2 | temporary | pg_default | 0000020000000047 | 0000020000000047 | 0000020000100000 | | +(3 rows) + +commit; +drop view undo_logs; diff --git a/src/test/modules/test_undo/sql/.gitignore b/src/test/modules/test_undo/sql/.gitignore new file mode 100644 index 00000000000..1bb8bf6d7fd --- /dev/null +++ b/src/test/modules/test_undo/sql/.gitignore @@ -0,0 +1 @@ +# empty diff --git a/src/test/modules/test_undo/test_undo--1.0.sql b/src/test/modules/test_undo/test_undo--1.0.sql new file mode 100644 index 00000000000..4ab4813cf44 --- /dev/null +++ b/src/test/modules/test_undo/test_undo--1.0.sql @@ -0,0 +1,53 @@ +\echo Use "CREATE EXTENSION test_undo" to load this file. \quit + +CREATE FUNCTION undo_allocate(size int, persistence text) +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_advance(ptr text, size int, persistence text) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_append(bytes bytea, persistence text) +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_append_transaction_header(xid xid, persistence text) +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_append_file(path text) +RETURNS text +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_extract_file(path text, undo_ptr text, size int, persistence text) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_dump(undo_ptr text, size int, persistence text) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_discard(undo_ptr text) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_is_discarded(undo_ptr text) +RETURNS boolean +AS 'MODULE_PATHNAME' +LANGUAGE C; + +CREATE FUNCTION undo_foreground_discard_test(loops int, size int, persistence text) +RETURNS void +AS 'MODULE_PATHNAME' +LANGUAGE C; + + diff --git a/src/test/modules/test_undo/test_undo.c b/src/test/modules/test_undo/test_undo.c new file mode 100644 index 00000000000..2296ba08654 --- /dev/null +++ b/src/test/modules/test_undo/test_undo.c @@ -0,0 +1,560 @@ +#include "postgres.h" + +#include "access/transam.h" +#include "access/undolog.h" +#include "catalog/pg_class.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "storage/bufmgr.h" +#include "utils/builtins.h" + +#include +#include + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(undo_allocate); +PG_FUNCTION_INFO_V1(undo_advance); +PG_FUNCTION_INFO_V1(undo_append); +PG_FUNCTION_INFO_V1(undo_append_file); +PG_FUNCTION_INFO_V1(undo_append_transaction_header); +PG_FUNCTION_INFO_V1(undo_extract_file); +PG_FUNCTION_INFO_V1(undo_dump); +PG_FUNCTION_INFO_V1(undo_discard); +PG_FUNCTION_INFO_V1(undo_is_discarded); +PG_FUNCTION_INFO_V1(undo_foreground_discard_test); + +/* + * It's nice to show UndoRecPtr always as hex, because that way you can see + * the components easily. Bigint just doesn't really work because it's + * signed. + */ +static text * +undo_rec_ptr_to_text(UndoRecPtr undo_ptr) +{ + char buffer[17]; + + snprintf(buffer, sizeof(buffer), UndoRecPtrFormat, undo_ptr); + return cstring_to_text(buffer); +} + +static UndoRecPtr +undo_rec_ptr_from_text(text *t) +{ + UndoRecPtr undo_ptr; + + if (sscanf(text_to_cstring(t), "%zx", &undo_ptr) != 1) + elog(ERROR, "could not parse UndoRecPtr (expected hex)"); + return undo_ptr; +} + +static UndoPersistence +undo_persistence_from_text(text *t) +{ + char *str = text_to_cstring(t); + + if (strcmp(str, "permanent") == 0) + return UNDO_PERMANENT; + else if (strcmp(str, "temporary") == 0) + return UNDO_TEMP; + else if (strcmp(str, "unlogged") == 0) + return UNDO_UNLOGGED; + else + elog(ERROR, "unknown undo persistence level: %s", str); +} + +/* + * Just allocate some undo space, for testing. This may cause us to be + * attached to an undo log, possibly creating it on demand. + */ +Datum +undo_allocate(PG_FUNCTION_ARGS) +{ + int size = PG_GETARG_INT32(0); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(1)); + UndoRecPtr undo_ptr; + + undo_ptr = UndoLogAllocate(size, persistence, NULL); + + PG_RETURN_TEXT_P(undo_rec_ptr_to_text(undo_ptr)); +} + +/* + * Advance the insert pointer for an undo log, for testing. This must + * undo_ptr value give must have been returned by undo_allocate(), and the + * size give must be the argument that was given to undo_allocate(). The call + * to undo_allocate() reserved space for us and told us where it is, and now + * we are advancing the insertion pointer (presumably having written data + * there). + */ +Datum +undo_advance(PG_FUNCTION_ARGS) +{ + UndoRecPtr undo_ptr = undo_rec_ptr_from_text(PG_GETARG_TEXT_PP(0)); + int size = PG_GETARG_INT32(1); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(2)); + + UndoLogAdvance(undo_ptr, size, persistence); + + PG_RETURN_VOID(); +} + +/* + * Advance the discard pointer in an undo log. + */ +Datum +undo_discard(PG_FUNCTION_ARGS) +{ + UndoRecPtr undo_ptr = undo_rec_ptr_from_text(PG_GETARG_TEXT_PP(0)); + + UndoLogDiscard(undo_ptr, InvalidTransactionId); + + PG_RETURN_VOID(); +} + +/* + * Allocate space and write the contents of a file into it. + */ +Datum +undo_append_file(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(1)); + size_t size; + size_t remaining; + UndoRecPtr start_undo_ptr; + UndoRecPtr insert_undo_ptr; + int fd; + + /* Open the file and check its size. */ + fd = open(path, O_RDONLY, 0); + if (fd < 0) + elog(ERROR, "could not open file '%s': %m", path); + size = lseek(fd, 0, SEEK_END); + lseek(fd, 0, SEEK_SET); + + /* Allocate undo log space. */ + start_undo_ptr = UndoLogAllocate(size, persistence, NULL); + + elog(NOTICE, "will copy %zu bytes into undo log", size); + + /* Copy data into shared buffers. */ + insert_undo_ptr = start_undo_ptr; + remaining = size; + while (remaining > 0) + { + RelFileNode rfn; + Buffer buffer; + char *page; + size_t this_chunk_offset; + size_t this_chunk_size; + char data[BLCKSZ]; + ssize_t bytes_read; + + /* + * Figure out how much we can fit on the page that insert_undo_ptr + * points to. + */ + this_chunk_offset = UndoRecPtrGetPageOffset(insert_undo_ptr); + this_chunk_size = Min(remaining, BLCKSZ - this_chunk_offset); + + Assert(this_chunk_offset >= UndoLogBlockHeaderSize); + Assert(this_chunk_size <= UndoLogUsableBytesPerPage); + Assert(this_chunk_offset + this_chunk_size <= BLCKSZ); + + bytes_read = read(fd, data, this_chunk_size); + if (bytes_read < 0) + { + int save_errno = errno; + close(fd); + errno = save_errno; + elog(ERROR, "failed to read from '%s': %m", path); + } + if (bytes_read < this_chunk_size) + { + /* + * This is a bit silly, we should be prepared to handle this but + * for this demo code we'll just give up. + */ + close(fd); + elog(ERROR, "short read from '%s'", path); + } + + /* Copy the chunk onto the page. */ + UndoRecPtrAssignRelFileNode(rfn, insert_undo_ptr); + buffer = + ReadBufferWithoutRelcache(rfn, + UndoLogForkNum, + UndoRecPtrGetBlockNum(insert_undo_ptr), + RBM_NORMAL, + NULL, + persistence); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + if (this_chunk_offset == UndoLogBlockHeaderSize) + PageInit(page, BLCKSZ, 0); + memcpy(page + this_chunk_offset, data, this_chunk_size); + MarkBufferDirty(buffer); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + + /* Prepare to put the next chunk on the next page. */ + insert_undo_ptr += this_chunk_size; + remaining -= this_chunk_size; + + /* Step over the page header if we landed at the start of page. */ + if (UndoRecPtrGetPageOffset(insert_undo_ptr) == 0) + insert_undo_ptr += UndoLogBlockHeaderSize; + } + + /* Advance the undo log insert point. No need to consider headers. */ + UndoLogAdvance(start_undo_ptr, size, persistence); + + /* + * We'd leak a file descriptor if code above raised an error, but not + * worrying about that for this demo code. + */ + close(fd); + + PG_RETURN_TEXT_P(undo_rec_ptr_to_text(start_undo_ptr)); +} + +/* + * Extract the contents of an undo log into a file. + */ +Datum +undo_extract_file(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + UndoRecPtr undo_ptr = undo_rec_ptr_from_text(PG_GETARG_TEXT_PP(1)); + size_t size = (size_t) PG_GETARG_INT32(2); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(3)); + size_t remaining = size; + int fd; + + if (UndoRecPtrGetPageOffset(undo_ptr) < UndoLogBlockHeaderSize) + elog(ERROR, "undo pointer points to header data"); + + fd = open(path, O_WRONLY | O_CREAT, 0664); + if (fd < 0) + elog(ERROR, "can't open '%s': %m", path); + + while (remaining > 0) + { + RelFileNode rfn; + Buffer buffer; + char *page; + size_t this_chunk_offset; + size_t this_chunk_size; + char data[BLCKSZ]; + ssize_t bytes_written; + + /* + * Figure out how much we can read from the page that undo_ptr points + * to. + */ + this_chunk_offset = UndoRecPtrGetPageOffset(undo_ptr); + this_chunk_size = Min(remaining, BLCKSZ - this_chunk_offset); + + /* Copy region of page contents to buffer. */ + UndoRecPtrAssignRelFileNode(rfn, undo_ptr); + buffer = + ReadBufferWithoutRelcache(rfn, + UndoLogForkNum, + UndoRecPtrGetBlockNum(undo_ptr), + RBM_NORMAL, + NULL, + persistence); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + memcpy(data, page + this_chunk_offset, this_chunk_size); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + + /* Write out. */ + bytes_written = write(fd, data, this_chunk_size); + if (bytes_written < 0) + { + int save_errno = errno; + close(fd); + errno = save_errno; + elog(ERROR, "failed to write to '%s': %m", path); + } + if (bytes_written < this_chunk_size) + { + /* + * This is a bit silly, we should be prepared to handle this but + * for this demo code we'll just give up. + */ + close(fd); + elog(ERROR, "short write to '%s'", path); + } + + /* Prepare to put the next chunk on the next page. */ + undo_ptr += this_chunk_size; + remaining -= this_chunk_size; + + /* Step over the page header if we landed at the start of page. */ + if (UndoRecPtrGetPageOffset(undo_ptr) == 0) + undo_ptr += UndoLogBlockHeaderSize; + } + PG_RETURN_VOID(); +} + +/* + * Allocate space and write data into it. + */ +static UndoRecPtr +undo_append_raw(void *data, size_t size, UndoPersistence persistence) +{ + size_t remaining; + UndoRecPtr start_undo_ptr; + UndoRecPtr insert_undo_ptr; + + /* Allocate undo log space for our data. */ + start_undo_ptr = UndoLogAllocate(size, persistence, NULL); + + elog(NOTICE, "will copy %zu bytes into undo log at " UndoRecPtrFormat, + size, start_undo_ptr); + + /* + * Copy data into shared buffers. Real code that does this would need to + * WAL-log something that would redo this. + */ + insert_undo_ptr = start_undo_ptr; + remaining = size; + while (remaining > 0) + { + RelFileNode rfn; + Buffer buffer; + char *page; + size_t this_chunk_offset; + size_t this_chunk_size; + + /* + * Figure out how much we can fit on the page that insert_undo_ptr + * points to. + */ + this_chunk_offset = UndoRecPtrGetPageOffset(insert_undo_ptr); + this_chunk_size = Min(remaining, BLCKSZ - this_chunk_offset); + + Assert(this_chunk_offset >= UndoLogBlockHeaderSize); + Assert(this_chunk_size <= UndoLogUsableBytesPerPage); + Assert(this_chunk_offset + this_chunk_size <= BLCKSZ); + elog(NOTICE, "writing chunk at offset %zu", this_chunk_offset); + + /* Copy the chunk onto the page. */ + UndoRecPtrAssignRelFileNode(rfn, insert_undo_ptr); + buffer = + ReadBufferWithoutRelcache(rfn, + UndoLogForkNum, + UndoRecPtrGetBlockNum(insert_undo_ptr), + RBM_NORMAL, + NULL, + persistence); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = BufferGetPage(buffer); + if (this_chunk_offset == UndoLogBlockHeaderSize) + PageInit(page, BLCKSZ, 0); + memcpy(page + this_chunk_offset, data, this_chunk_size); + MarkBufferDirty(buffer); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + + /* Prepare to put the next chunk on the next page. */ + insert_undo_ptr += this_chunk_size; + data = (char *) data + this_chunk_size; + remaining -= this_chunk_size; + + /* Step over the page header if we landed at the start of page. */ + if (UndoRecPtrGetPageOffset(insert_undo_ptr) == 0) + insert_undo_ptr += UndoLogBlockHeaderSize; + } + + /* Advance the undo log insert point. No need to consider headers. */ + UndoLogAdvance(start_undo_ptr, size, persistence); + + return start_undo_ptr; +} + +/* + * Allocate space and write data into it. + */ +Datum +undo_append(PG_FUNCTION_ARGS) +{ + bytea *input = PG_GETARG_BYTEA_PP(0); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(1)); + void *data = VARDATA_ANY(input); + size_t size = VARSIZE_ANY_EXHDR(input); + + PG_RETURN_TEXT_P(undo_rec_ptr_to_text(undo_append_raw(data, size, persistence))); +} + + +/* + * We need to be able to write a transaction header that will prevent the undo + * background worker from discarding any data that follows it until the + * referenced xid has committed. We define this here to avoid problematic + * interactions with later patches that add record level abstractions, but it + * might be removed later. + */ +typedef struct TestRecordHeader +{ + uint8 urec_type; + uint8 urec_info; + uint16 urec_prevlen; + Oid urec_relfilenode; + TransactionId urec_prevxid; + TransactionId urec_xid; + CommandId urec_cid; +} TestRecordHeader; + +typedef struct TestRecordTransaction +{ + uint32 urec_xidepoch; + uint64 urec_next; +} TestRecordTransaction; + +Datum +undo_append_transaction_header(PG_FUNCTION_ARGS) +{ + TestRecordHeader header1; + TestRecordTransaction header2; + TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0)); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(1)); + UndoRecPtr result; + + memset(&header1, 0, sizeof(header1)); + header1.urec_type = 0x08; + header1.urec_xid = xid; + memset(&header2, 0, sizeof(header2)); + header2.urec_next = InvalidUndoRecPtr; + + result = + undo_append_raw(&header1, + offsetof(TestRecordHeader, urec_cid) + + sizeof(CommandId), + persistence); + undo_append_raw(&header2, sizeof(header2), persistence); + + PG_RETURN_TEXT_P(undo_rec_ptr_to_text(result)); +} + +Datum +undo_dump(PG_FUNCTION_ARGS) +{ + UndoRecPtr undo_ptr = undo_rec_ptr_from_text(PG_GETARG_TEXT_PP(0)); + size_t size = (size_t) PG_GETARG_INT32(1); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(2)); + size_t remaining; + + + /* Rewind so that we start on an 8-byte block. */ + if (undo_ptr % 8 != 0) + { + int extra_prefix = 8 - undo_ptr % 8; + + undo_ptr -= extra_prefix; + size += extra_prefix; + } + /* Extend size so we show an 8-byte block. */ + if (size % 8 != 0) + size += 8 - size % 8; + remaining = size; + + while (remaining > 0) + { + RelFileNode rfn; + Buffer buffer; + char *page; + size_t this_chunk_offset; + size_t this_chunk_size; + unsigned char data[8]; + char line[80]; + int i; + + /* + * Figure out how much we can read from the page that undo_ptr points + * to. + */ + this_chunk_offset = UndoRecPtrGetPageOffset(undo_ptr); + this_chunk_size = 8; + + /* Copy region of page contents to buffer. */ + UndoRecPtrAssignRelFileNode(rfn, undo_ptr); + buffer = + ReadBufferWithoutRelcache(rfn, + UndoLogForkNum, + UndoRecPtrGetBlockNum(undo_ptr), + RBM_NORMAL, + NULL, + persistence); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + page = BufferGetPage(buffer); + memcpy(data, page + this_chunk_offset, this_chunk_size); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buffer); + + /* Write out. Apologies for this horrible code. */ + snprintf(line, sizeof(line), UndoRecPtrFormat ": ", undo_ptr); + for (i = 0; i < 8; ++i) + snprintf(&line[18 + 3 * i], 4, "%02x ", data[i]); + for (i = 0; i < 8; ++i) + { + char c = '.'; + + if (data[i] >= ' ' && data[i] <= 127) + c = data[i]; + line[18 + 3 * 8 + i] = c; + } + line[18 + 3 * i + i] = '\0'; + elog(NOTICE, "%s", line); + + /* Prepare to put the next chunk on the next page. */ + undo_ptr += this_chunk_size; + remaining -= this_chunk_size; + + /* Step over the page header if we landed at the start of page. */ + if (UndoRecPtrGetPageOffset(undo_ptr) == 0) + undo_ptr += UndoLogBlockHeaderSize; + } + PG_RETURN_VOID(); +} + +Datum +undo_foreground_discard_test(PG_FUNCTION_ARGS) +{ + int loops = PG_GETARG_INT32(0); + int size = PG_GETARG_INT32(1); + UndoPersistence persistence = undo_persistence_from_text(PG_GETARG_TEXT_PP(2)); + int i; + + if (size > BLCKSZ) + elog(ERROR, "data too large"); + + for (i = 0; i < loops; ++i) + { + UndoRecPtr undo_ptr; + + /* Allocate some space. */ + undo_ptr = UndoLogAllocate(size, persistence, NULL); + UndoLogAdvance(undo_ptr, size, persistence); + + /* Discard the space that we just allocated. */ + UndoLogDiscard(undo_ptr + size, InvalidTransactionId); + } + + PG_RETURN_VOID(); +} + +/* + * Check if an undo pointer has been discarded. + */ +Datum +undo_is_discarded(PG_FUNCTION_ARGS) +{ + PG_RETURN_BOOL(UndoLogIsDiscarded(undo_rec_ptr_from_text(PG_GETARG_TEXT_PP(0)))); +} diff --git a/src/test/modules/test_undo/test_undo.control b/src/test/modules/test_undo/test_undo.control new file mode 100644 index 00000000000..4595f52477b --- /dev/null +++ b/src/test/modules/test_undo/test_undo.control @@ -0,0 +1,4 @@ +comment = 'test_undo' +default_version = '1.0' +module_pathname = '$libdir/test_undo' +relocatable = true -- 2.17.0