From a86a01be7f35d2f71cdfc3e887d29e2d5edf820a Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Wed, 25 Apr 2018 13:00:28 +0300 Subject: [PATCH] Rewrite unused_oids in perl unused_oids script was broken with bsd sed after 372728b0d49. It was possible to write more portable regexps but it seems easier to just rewrite unused_oids in perl to match duplicate_oids. Also add in-place complain about duplicates instead of running uniq through oids array. --- src/backend/catalog/Catalog.pm | 47 ++++++++++++++++++++++++++++ src/include/catalog/duplicate_oids | 23 +++++--------- src/include/catalog/unused_oids | 63 ++++++++++++++++++-------------------- 3 files changed, 85 insertions(+), 48 deletions(-) diff --git a/src/backend/catalog/Catalog.pm b/src/backend/catalog/Catalog.pm index 6305a2b362..5caf428b6a 100644 --- a/src/backend/catalog/Catalog.pm +++ b/src/backend/catalog/Catalog.pm @@ -40,7 +40,9 @@ sub ParseHeader $catalog{columns} = []; $catalog{toasting} = []; + $catalog{toasting_oids} = []; $catalog{indexing} = []; + $catalog{indexing_oids} = []; $catalog{client_code} = []; open(my $ifh, '<', $input_file) || die "$input_file: $!"; @@ -89,6 +91,7 @@ sub ParseHeader my ($toast_name, $toast_oid, $index_oid) = ($1, $2, $3); push @{ $catalog{toasting} }, "declare toast $toast_oid $index_oid on $toast_name\n"; + push @{ $catalog{toasting_oids} }, ($toast_oid, $index_oid); } elsif (/^DECLARE_(UNIQUE_)?INDEX\(\s*(\w+),\s*(\d+),\s*(.+)\)/) { @@ -99,6 +102,7 @@ sub ParseHeader "declare %sindex %s %s %s\n", $is_unique ? 'unique ' : '', $index_name, $index_oid, $using); + push @{ $catalog{indexing_oids} }, ($index_oid); } elsif (/^BUILD_INDICES/) { @@ -389,4 +393,47 @@ sub FindDefinedSymbolFromData die "no definition found for $symbol\n"; } +sub FindAllOidsFromHeaders +{ + my @input_files = @_; + my @oids = (); + + foreach my $header (@input_files) + { + $header =~ /(.+)\.h$/ + or die "Input files need to be header files.\n"; + my $datfile = "$1.dat"; + + my $catalog = Catalog::ParseHeader($header); + + push @oids, $catalog->{rowtype_oid} if ($catalog->{rowtype_oid}); + push @oids, $catalog->{relation_oid} if ($catalog->{relation_oid}); + + # pg_class row oids are duplicated in typrelid of pg_type + if (exists($catalog->{catname}) and ($catalog->{catname} eq 'pg_class')) + { + next; + } + + if (-e $datfile) + { + my $catdata = Catalog::ParseData($datfile, $catalog->{columns}, 0); + foreach my $row (@$catdata) + { + # filter entries for bootstrapped catalogs entries in pg_type + if (($catalog->{catname} eq 'pg_type') and ($row->{typcategory} eq 'C')) + { + next; + } + push @oids, $row->{oid} if defined $row->{oid}; + } + } + + push @oids, @{ $catalog->{toasting_oids} }; + push @oids, @{ $catalog->{indexing_oids} }; + } + + return \@oids; +} + 1; diff --git a/src/include/catalog/duplicate_oids b/src/include/catalog/duplicate_oids index 8c143cf06f..c818dc8a67 100755 --- a/src/include/catalog/duplicate_oids +++ b/src/include/catalog/duplicate_oids @@ -1,27 +1,20 @@ #!/usr/bin/perl +use lib '../../backend/catalog/'; +use Catalog; + use strict; use warnings; -BEGIN -{ - @ARGV = (glob("pg_*.h"), glob("pg_*.dat"), qw(indexing.h toasting.h)); -} +my @input_files = (glob("pg_*.h"), qw(indexing.h toasting.h)); + +my @oids = @{ Catalog::FindAllOidsFromHeaders(@input_files) }; my %oidcounts; -while (<>) +foreach my $oid (@oids) { - next if /^CATALOG\(.*BKI_BOOTSTRAP/; - next - unless /\boid *=> *'(\d+)'/ - || /^CATALOG\([^,]*, *(\d+).*BKI_ROWTYPE_OID\((\d+),/ - || /^CATALOG\([^,]*, *(\d+)/ - || /^DECLARE_INDEX\([^,]*, *(\d+)/ - || /^DECLARE_UNIQUE_INDEX\([^,]*, *(\d+)/ - || /^DECLARE_TOAST\([^,]*, *(\d+), *(\d+)/; - $oidcounts{$1}++; - $oidcounts{$2}++ if $2; + $oidcounts{$oid}++; } my $found = 0; diff --git a/src/include/catalog/unused_oids b/src/include/catalog/unused_oids index f71222d50d..4b0e16cc02 100755 --- a/src/include/catalog/unused_oids +++ b/src/include/catalog/unused_oids @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/perl # # unused_oids # @@ -15,43 +15,40 @@ # # run this script in src/include/catalog. # +use lib '../../backend/catalog/'; +use Catalog; +use strict; +use warnings; -AWK="awk" +my @input_files = (glob("pg_*.h"), qw(indexing.h toasting.h)); +my @oids; -# Get FirstBootstrapObjectId from access/transam.h -FIRSTOBJECTID=`grep '#define[ ]*FirstBootstrapObjectId' ../access/transam.h | $AWK '{ print $3 }'` -export FIRSTOBJECTID +push @oids, @{ Catalog::FindAllOidsFromHeaders(@input_files) }; -# this part (down to the uniq step) should match the duplicate_oids script -# note: we exclude BKI_BOOTSTRAP relations since they are expected to have -# matching data entries in pg_class.dat and pg_type.dat +my $FirstBootstrapObjectId = Catalog::FindDefinedSymbol( + 'access/transam.h', [".."], 'FirstBootstrapObjectId'); +push @oids, $FirstBootstrapObjectId; -cat pg_*.h pg_*.dat toasting.h indexing.h | -egrep -v -e '^CATALOG\(.*BKI_BOOTSTRAP' | \ -sed -n -e 's/.*\boid *=> *'\''\([0-9][0-9]*\)'\''.*$/\1/p' \ - -e 's/^CATALOG([^,]*, *\([0-9][0-9]*\).*BKI_ROWTYPE_OID(\([0-9][0-9]*\),.*$/\1,\2/p' \ - -e 's/^CATALOG([^,]*, *\([0-9][0-9]*\).*$/\1/p' \ - -e 's/^DECLARE_INDEX([^,]*, *\([0-9][0-9]*\).*$/\1/p' \ - -e 's/^DECLARE_UNIQUE_INDEX([^,]*, *\([0-9][0-9]*\).*$/\1/p' \ - -e 's/^DECLARE_TOAST([^,]*, *\([0-9][0-9]*\), *\([0-9][0-9]*\).*$/\1,\2/p' | \ -tr ',' '\n' | \ -sort -n | \ -uniq | \ -$AWK ' -BEGIN { - last = 0; -} -/^[0-9]/ { - if ($1 > last + 1) { - if ($1 > last + 2) { - print last + 1, "-", $1 - 1; - } else { - print last + 1; + +my $prev_oid = 0; +foreach my $oid (sort { $a <=> $b } @oids) +{ + if ($oid > $prev_oid + 1) + { + if ($oid > $prev_oid + 2) + { + print "@{[ $prev_oid + 1 ]} - @{[ $oid - 1 ]}\n"; } + else + { + print "@{[ $prev_oid + 1 ]}\n"; + } + } + elsif ($oid == $prev_oid) + { + print "Duplicate oid detected: $oid\n"; + exit 1; } - last = $1; + $prev_oid = $oid; } -END { - print last + 1, "-", ENVIRON["FIRSTOBJECTID"]-1; -}' -- 2.16.2