From 9cf5c517b6a9359fb1863e3315f5412860db45ff Mon Sep 17 00:00:00 2001 From: John Naylor Date: Thu, 26 Apr 2018 15:54:53 +0700 Subject: [PATCH 1/2] Remove standalone script duplicate_oids Commit 5602265f770 added duplicate checking to unused_oids, so use that script for manual checks. Also change it so that it doesn't exit when encountering a duplicate -- doing so slows down fixing multiple duplicates. This left unused_oids as the only caller of FindAllOidsFromHeaders(). Since that function no longer offers any useful abstraction, remove it from Catalog.pm and inline it into unused_oids. Make genbki.pl responsible for compile time duplicate checks. This simplifies the Makefile a bit and keeps from having to read the catalog files twice. --- doc/src/sgml/bki.sgml | 9 +++---- src/backend/catalog/Catalog.pm | 51 -------------------------------------- src/backend/catalog/Makefile | 3 +-- src/backend/catalog/genbki.pl | 31 ++++++++++++++++++++++- src/include/catalog/duplicate_oids | 29 ---------------------- src/include/catalog/unused_oids | 49 +++++++++++++++++++++++++++++++++--- 6 files changed, 80 insertions(+), 92 deletions(-) delete mode 100755 src/include/catalog/duplicate_oids diff --git a/doc/src/sgml/bki.sgml b/doc/src/sgml/bki.sgml index 5a4cd39..087272c 100644 --- a/doc/src/sgml/bki.sgml +++ b/doc/src/sgml/bki.sgml @@ -377,13 +377,12 @@ script src/include/catalog/unused_oids. It prints inclusive ranges of unused OIDs (e.g., the output line 45-900 means OIDs 45 through 900 have not been - allocated yet). Currently, OIDs 1-9999 are reserved for manual + allocated yet), as well as any duplicate OIDs found. + Currently, OIDs 1-9999 are reserved for manual assignment; the unused_oids script simply looks through the catalog headers and .dat files - to see which ones do not appear. You can also use - the duplicate_oids script to check for mistakes. - (That script is run automatically at compile time, and will stop the - build if a duplicate is found.) + to see which ones do not appear. In addition, if genbki.pl detects + duplicates at compile time, it will stop the build. diff --git a/src/backend/catalog/Catalog.pm b/src/backend/catalog/Catalog.pm index 0b057a8..823e09a 100644 --- a/src/backend/catalog/Catalog.pm +++ b/src/backend/catalog/Catalog.pm @@ -384,55 +384,4 @@ sub FindDefinedSymbolFromData die "no definition found for $symbol\n"; } -# Extract an array of all the OIDs assigned in the specified catalog headers -# and their associated data files (if any). -sub FindAllOidsFromHeaders -{ - my @input_files = @_; - - my @oids = (); - - foreach my $header (@input_files) - { - $header =~ /(.+)\.h$/ - or die "Input files need to be header files.\n"; - my $datfile = "$1.dat"; - - my $catalog = Catalog::ParseHeader($header); - - # We ignore the pg_class OID and rowtype OID of bootstrap catalogs, - # as those are expected to appear in the initial data for pg_class - # and pg_type. For regular catalogs, include these OIDs. - if (!$catalog->{bootstrap}) - { - push @oids, $catalog->{relation_oid} - if ($catalog->{relation_oid}); - push @oids, $catalog->{rowtype_oid} if ($catalog->{rowtype_oid}); - } - - # Not all catalogs have a data file. - if (-e $datfile) - { - my $catdata = - Catalog::ParseData($datfile, $catalog->{columns}, 0); - - foreach my $row (@$catdata) - { - push @oids, $row->{oid} if defined $row->{oid}; - } - } - - foreach my $toast (@{ $catalog->{toasting} }) - { - push @oids, $toast->{toast_oid}, $toast->{toast_index_oid}; - } - foreach my $index (@{ $catalog->{indexing} }) - { - push @oids, $index->{index_oid}; - } - } - - return \@oids; -} - 1; diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index d25d98a..a54197d 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -84,8 +84,7 @@ generated-header-symlinks: $(top_builddir)/src/include/catalog/header-stamp # configure run, even in distribution tarballs. So depending on configure.in # instead is cheating a bit, but it will achieve the goal of updating the # version number when it changes. -bki-stamp: genbki.pl Catalog.pm $(POSTGRES_BKI_SRCS) $(POSTGRES_BKI_DATA) $(top_srcdir)/configure.in $(top_srcdir)/src/include/catalog/duplicate_oids - cd $(top_srcdir)/src/include/catalog && $(PERL) ./duplicate_oids +bki-stamp: genbki.pl Catalog.pm $(POSTGRES_BKI_SRCS) $(POSTGRES_BKI_DATA) $(top_srcdir)/configure.in $(PERL) -I $(catalogdir) $< --set-version=$(MAJORVERSION) $(POSTGRES_BKI_SRCS) touch $@ diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index 83b6158..199a068 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -78,6 +78,7 @@ my %catalogs; my %catalog_data; my @toast_decls; my @index_decls; +my %oidcounts; # number of times we see an OID foreach my $header (@input_files) { $header =~ /(.+)\.h$/ @@ -94,10 +95,27 @@ foreach my $header (@input_files) $catalogs{$catname} = $catalog; } + # We ignore the pg_class OID and rowtype OID of bootstrap catalogs, + # as those are expected to appear in the initial data for pg_class + # and pg_type. For regular catalogs, include these OIDs. + if (!$catalog->{bootstrap}) + { + $oidcounts{ $catalog->{relation_oid} }++ + if ($catalog->{relation_oid}); + $oidcounts{ $catalog->{rowtype_oid} }++ + if ($catalog->{rowtype_oid}); + } + # Not all catalogs have a data file. if (-e $datfile) { - $catalog_data{$catname} = Catalog::ParseData($datfile, $schema, 0); + my $data = Catalog::ParseData($datfile, $schema, 0); + $catalog_data{$catname} = $data; + foreach my $row (@$data) + { + $oidcounts{ $row->{oid} }++ if defined $row->{oid}; + } + } # If the header file contained toast or index info, build BKI @@ -119,6 +137,17 @@ foreach my $header (@input_files) } } +# Exit if we have any duplicate OIDs. +my $found = 0; +foreach my $oid (keys %oidcounts) +{ + next unless $oidcounts{$oid} > 1; + print "Duplicate oids detected:\n" if !$found; + $found = 1; + print "$oid\n"; +} +die if $found; + # Fetch some special data that we will substitute into the output file. # CAUTION: be wary about what symbols you substitute into the .bki file here! # It's okay to substitute things that are expected to be really constant diff --git a/src/include/catalog/duplicate_oids b/src/include/catalog/duplicate_oids deleted file mode 100755 index 0d7aa15..0000000 --- a/src/include/catalog/duplicate_oids +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/perl - -use lib '../../backend/catalog/'; -use Catalog; - -use strict; -use warnings; - -my @input_files = (glob("pg_*.h"), qw(indexing.h toasting.h)); - -my $oids = Catalog::FindAllOidsFromHeaders(@input_files); - -my %oidcounts; - -foreach my $oid (@{$oids}) -{ - $oidcounts{$oid}++; -} - -my $found = 0; - -foreach my $oid (sort { $a <=> $b } keys %oidcounts) -{ - next unless $oidcounts{$oid} > 1; - $found = 1; - print "$oid\n"; -} - -exit $found; diff --git a/src/include/catalog/unused_oids b/src/include/catalog/unused_oids index a727225..333b547 100755 --- a/src/include/catalog/unused_oids +++ b/src/include/catalog/unused_oids @@ -23,16 +23,58 @@ use warnings; my @input_files = (glob("pg_*.h"), qw(indexing.h toasting.h)); -my $oids = Catalog::FindAllOidsFromHeaders(@input_files); +# Extract an array of all the OIDs assigned in the specified catalog headers +# and their associated data files (if any). +my @oids; +foreach my $header (@input_files) +{ + $header =~ /(.+)\.h$/ + or die "Input files need to be header files.\n"; + my $datfile = "$1.dat"; + + my $catalog = Catalog::ParseHeader($header); + + # We ignore the pg_class OID and rowtype OID of bootstrap catalogs, + # as those are expected to appear in the initial data for pg_class + # and pg_type. For regular catalogs, include these OIDs. + if (!$catalog->{bootstrap}) + { + push @oids, $catalog->{relation_oid} + if ($catalog->{relation_oid}); + push @oids, $catalog->{rowtype_oid} + if ($catalog->{rowtype_oid}); + } + + # Not all catalogs have a data file. + if (-e $datfile) + { + my $catdata = + Catalog::ParseData($datfile, $catalog->{columns}, 0); + + foreach my $row (@$catdata) + { + push @oids, $row->{oid} if defined $row->{oid}; + } + } + + foreach my $toast (@{ $catalog->{toasting} }) + { + push @oids, $toast->{toast_oid}, $toast->{toast_index_oid}; + } + foreach my $index (@{ $catalog->{indexing} }) + { + push @oids, $index->{index_oid}; + } +} # Also push FirstBootstrapObjectId to serve as a terminator for the last gap. my $FirstBootstrapObjectId = Catalog::FindDefinedSymbol('access/transam.h', [".."], 'FirstBootstrapObjectId'); -push @{$oids}, $FirstBootstrapObjectId; +push @oids, $FirstBootstrapObjectId; my $prev_oid = 0; -foreach my $oid (sort { $a <=> $b } @{$oids}) +foreach my $oid (sort { $a <=> $b } @oids) { if ($oid > $prev_oid + 1) { @@ -48,7 +90,6 @@ foreach my $oid (sort { $a <=> $b } @{$oids}) elsif ($oid == $prev_oid) { print "Duplicate oid detected: $oid\n"; - exit 1; } $prev_oid = $oid; } -- 2.7.4