From 386cfe747bc4ccd867f3e27f5f7669c8eb7692f3 Mon Sep 17 00:00:00 2001 From: Maxim Orlov Date: Sat, 2 Nov 2024 10:46:16 +0300 Subject: [PATCH v6 6/6] TEST: add basic mxidoff64 tests 005_mxidoff.pl --- src/bin/pg_upgrade/t/005_mxidoff.pl | 389 ++++++++++++++++++++++++++++ 1 file changed, 389 insertions(+) create mode 100644 src/bin/pg_upgrade/t/005_mxidoff.pl diff --git a/src/bin/pg_upgrade/t/005_mxidoff.pl b/src/bin/pg_upgrade/t/005_mxidoff.pl new file mode 100644 index 0000000000..e595870543 --- /dev/null +++ b/src/bin/pg_upgrade/t/005_mxidoff.pl @@ -0,0 +1,389 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; + +use File::Find qw(find); + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if (!defined($ENV{oldinstall})) +{ + die "oldinstall is not defined"; +} + +sub mxid_prepare +{ + my ($node) = @_; + + $node->safe_psql('postgres', + q( + CREATE TABLE FOO(BAR INT PRIMARY KEY, BAZ INT); + CREATE OR REPLACE PROCEDURE MXIDFILLER(N_STEPS INT DEFAULT 1000) + LANGUAGE PLPGSQL + AS $$ + BEGIN + FOR I IN 1..N_STEPS LOOP + UPDATE FOO SET BAZ = RANDOM(1, 1000) + WHERE BAR IN (SELECT BAR FROM FOO TABLESAMPLE BERNOULLI(80)); + COMMIT; + END LOOP; + END;$$; + INSERT INTO FOO (BAR, BAZ) SELECT ID, ID FROM GENERATE_SERIES(1, 512) ID; + )); +} + +sub mxid_fill +{ + my ($node) = @_; + + $node->safe_psql('postgres', + q( + BEGIN; + SELECT * FROM FOO FOR KEY SHARE; + PREPARE TRANSACTION 'A'; + CALL MXIDFILLER(365); + COMMIT PREPARED 'A'; + ), + timeout => 3600); +} + +# Fetch latest multixact checkpoint values. +sub multi_bounds +{ + my ($node) = @_; + my ($stdout, $stderr) = run_command([ 'pg_controldata', $node->data_dir ]); + my @control_data = split("\n", $stdout); + my $next = undef; + my $oldest = undef; + + foreach (@control_data) + { + if ($_ =~ /^Latest checkpoint's NextMultiXactId:\s*(.*)$/mg) + { + $next = $1; + } + + if ($_ =~ /^Latest checkpoint's oldestMultiXid:\s*(.*)$/mg) + { + $oldest = $1; + } + + if (defined($oldest) && defined($next)) + { + last; + } + } + + die "Latest checkpoint's NextMultiXactId not found in control file!\n" + unless defined($next); + + die "Latest checkpoint's oldestMultiXid not found in control file!\n" + unless defined($oldest); + + return ($oldest, $next); +} + +# List pg_multixact/offsets segments filenames. +sub list_actual_multixact_offsets +{ + my ($node) = @_; + my $dir; + + opendir($dir, $node->data_dir . '/pg_multixact/offsets') or die $!; + my @list = sort grep { /[0-9A-F]+/ } readdir $dir; + closedir $dir; + + return @list; +} + +use constant SIZEOF_MULTI_XACT_OFFSET => 8; +use constant BLCKSZ => 8192; +use constant MULTIXACT_OFFSETS_PER_PAGE => BLCKSZ / SIZEOF_MULTI_XACT_OFFSET; +use constant SLRU_PAGES_PER_SEGMENT => 2; + +# See src/backend/access/transam/multixact.c +sub MultiXactIdToOffsetSegment +{ + my ($multi) = @_; + + return $multi / MULTIXACT_OFFSETS_PER_PAGE / SLRU_PAGES_PER_SEGMENT; +} + +# Validate pg_multixact/offsets segments conversion. +sub validate_multixact_offsets +{ + my ($old, $new, $oldnode) = @_; + my ($oldest, $next) = multi_bounds($oldnode); + my $maxsegno = MultiXactIdToOffsetSegment($next); + my $maxsegname = sprintf("%04X", $maxsegno); + + print(">>>>>>>>>\n"); + foreach my $segname ( @$old ) + { + my $segno = hex($segname) * 2; + my $converted1 = sprintf("%04X", $segno); + my $converted2 = sprintf("%04X", $segno + 1); + + print "[${segname}] -> [${converted1}, ${converted2}] \n"; + # Skip the last segment as it may be incomplete. + if (not $converted1 eq $maxsegname) + { + die "Segmanet ${segname} is not properly converted" + unless (not $converted1 eq $maxsegname) and + grep { $converted1 eq $_ } @$new and + grep { $converted2 eq $_ } @$new; + } + } + print(">>>>>>>>>\n"); + + return 1; +} + +# +# Select tests to run. +# +my @tests = (0, 1, 2, 3); + +# ============================================================================= +# CASE 0 +# +# There must be several segments starting from the zero. +# ============================================================================= +SKIP: +{ + skip "case 0", 0 + unless ( grep( /^0$/, @tests ) ); + + my $oldnode = PostgreSQL::Test::Cluster->new('old_node0', + install_path => $ENV{oldinstall}); + $oldnode->init(force_initdb => 1); + $oldnode->append_conf('postgresql.conf', 'max_prepared_transactions = 2'); + $oldnode->append_conf('fsync', 'off'); + + my $newnode = PostgreSQL::Test::Cluster->new('new_node0'); + $newnode->init(); + + command_ok( + [ + 'pg_upgrade', '--no-sync', + '-d', $oldnode->data_dir, + '-D', $newnode->data_dir, + '-b', $oldnode->config_data('--bindir'), + '-B', $newnode->config_data('--bindir'), + '-s', $newnode->host, + '-p', $oldnode->port, + '-P', $newnode->port, + '--copy' + ], + 'run of pg_upgrade'); + + my @o = list_actual_multixact_offsets($oldnode); + my @n = list_actual_multixact_offsets($newnode); + ok(validate_multixact_offsets(\@o, \@n, $oldnode), + "case0: offsets segmants matched"); + + $oldnode->start(); + $newnode->start(); + + # just in case... + my $oldval = $oldnode->safe_psql('postgres', q(SELECT 1)); + my $newval = $newnode->safe_psql('postgres', q(SELECT 1)); + is($oldval, $newval, "case1: select eq"); + + $oldnode->stop(); + $newnode->stop(); +} + +# ============================================================================= +# CASE 1 +# +# There must be several segments starting from the zero. +# ============================================================================= +SKIP: +{ + skip "case 1", 1 + unless ( grep( /^1$/, @tests ) ); + + my $oldnode = PostgreSQL::Test::Cluster->new('old_node1', + install_path => $ENV{oldinstall}); + $oldnode->init(force_initdb => 1); + $oldnode->append_conf('postgresql.conf', 'max_prepared_transactions = 2'); + $oldnode->append_conf('fsync', 'off'); + $oldnode->start(); + + mxid_prepare($oldnode); + mxid_fill($oldnode); + + $oldnode->safe_psql('postgres', q(CHECKPOINT)); + $oldnode->stop(); + + my $newnode = PostgreSQL::Test::Cluster->new('new_node1'); + $newnode->init(); + + command_ok( + [ + 'pg_upgrade', '--no-sync', + '-d', $oldnode->data_dir, + '-D', $newnode->data_dir, + '-b', $oldnode->config_data('--bindir'), + '-B', $newnode->config_data('--bindir'), + '-s', $newnode->host, + '-p', $oldnode->port, + '-P', $newnode->port, + '--copy' + ], + 'run of pg_upgrade'); + + my @o = list_actual_multixact_offsets($oldnode); + my @n = list_actual_multixact_offsets($newnode); + ok(validate_multixact_offsets(\@o, \@n, $oldnode), + "case1: offsets segmants matched"); + + $oldnode->start(); + $newnode->start(); + + # just in case... + my $oldval = $oldnode->safe_psql('postgres', q(SELECT * FROM FOO)); + my $newval = $newnode->safe_psql('postgres', q(SELECT * FROM FOO)); + is($oldval, $newval, "case1: select eq"); + + $oldnode->stop(); + $newnode->stop(); +} + +# ============================================================================= +# CASE 2 +# +# Non-standard oldestMultiXid and NextMultiXactId. +# There must be several segments starting from some value. +# ============================================================================= +SKIP: +{ + skip "case 2", 2 + unless ( grep( /^2$/, @tests ) ); + + my $oldnode = PostgreSQL::Test::Cluster->new('old_node2', + install_path => $ENV{oldinstall}); + $oldnode->init(force_initdb => 1, + extra => [ + '-m', '0x123000', '-o', '0x123000' + ]); + + # Fixup MOX patch quirk + unlink $oldnode->data_dir . '/pg_multixact/members/0000'; + unlink $oldnode->data_dir . '/pg_multixact/offsets/0000'; + + $oldnode->append_conf('postgresql.conf', 'max_prepared_transactions = 2'); + $oldnode->append_conf('fsync', 'off'); + $oldnode->start(); + + mxid_prepare($oldnode); + mxid_fill($oldnode); + + $oldnode->safe_psql('postgres', q(CHECKPOINT)); + $oldnode->stop(); + + my $newnode = PostgreSQL::Test::Cluster->new('new_node2'); + $newnode->init(); + + command_ok( + [ + 'pg_upgrade', '--no-sync', + '-d', $oldnode->data_dir, + '-D', $newnode->data_dir, + '-b', $oldnode->config_data('--bindir'), + '-B', $newnode->config_data('--bindir'), + '-s', $newnode->host, + '-p', $oldnode->port, + '-P', $newnode->port, + '--copy' + ], + 'run of pg_upgrade'); + + my @o = list_actual_multixact_offsets($oldnode); + my @n = list_actual_multixact_offsets($newnode); + ok(validate_multixact_offsets(\@o, \@n, $oldnode), + "case2: non-standard offsets segmants matched"); + + $oldnode->start(); + $newnode->start(); + + # just in case... + my $oldval = $oldnode->safe_psql('postgres', q(SELECT * FROM FOO)); + my $newval = $newnode->safe_psql('postgres', q(SELECT * FROM FOO)); + is($oldval, $newval, "case2: select eq"); + + $oldnode->stop(); + $newnode->stop(); +} + +# ============================================================================= +# CASE 3 +# +# Non-standard oldestMultiXid and NextMultiXactId. +# ============================================================================= +SKIP: +{ + skip "case 3", 3 + unless ( grep( /^3$/, @tests ) ); + chdir ${PostgreSQL::Test::Utils::tmp_check}; + my $oldnode = PostgreSQL::Test::Cluster->new('old_node3', + install_path => $ENV{oldinstall}); + $oldnode->init(force_initdb => 1, + extra => [ + '-m', '0xFFFF0000', '-o', '0xFFFF0000' + ]); + + # Fixup MOX patch quirk + unlink $oldnode->data_dir . '/pg_multixact/members/0000'; + unlink $oldnode->data_dir . '/pg_multixact/offsets/0000'; + + $oldnode->append_conf('postgresql.conf', 'max_prepared_transactions = 2'); + $oldnode->append_conf('fsync', 'off'); + $oldnode->start(); + + mxid_prepare($oldnode); + mxid_fill($oldnode); + mxid_fill($oldnode); + + $oldnode->safe_psql('postgres', q(CHECKPOINT)); + $oldnode->stop(); + + my $newnode = PostgreSQL::Test::Cluster->new('new_node3'); + $newnode->init(); + + command_ok( + [ + 'pg_upgrade', '--no-sync', + '-d', $oldnode->data_dir, + '-D', $newnode->data_dir, + '-b', $oldnode->config_data('--bindir'), + '-B', $newnode->config_data('--bindir'), + '-s', $newnode->host, + '-p', $oldnode->port, + '-P', $newnode->port, + '--copy' + ], + 'run of pg_upgrade'); + + my @o = list_actual_multixact_offsets($oldnode); + my @n = list_actual_multixact_offsets($newnode); + ok(validate_multixact_offsets(\@o, \@n, $oldnode), + "case3: multi warp, non-standard offsets segmants matched"); + + $oldnode->start(); + $newnode->start(); + + # just in case... + my $oldval = $oldnode->safe_psql('postgres', q(SELECT * FROM FOO)); + my $newval = $newnode->safe_psql('postgres', q(SELECT * FROM FOO)); + is($oldval, $newval, "case3: select eq"); + + $oldnode->stop(); + $newnode->stop(); +} + +done_testing(); -- 2.43.0