From 3ea88b30bc696d7e80058e2f1ec1d2d5ad36bd7e Mon Sep 17 00:00:00 2001 From: Justin Pryzby Date: Thu, 3 Nov 2022 18:09:14 -0500 Subject: [PATCH 3/3] f!rewrite using a single filtering function --- src/bin/pg_dump/filter.c | 173 +++++++++++++++++- src/bin/pg_dump/filter.h | 71 +++++--- src/bin/pg_dump/pg_backup.h | 17 +- src/bin/pg_dump/pg_backup_archiver.c | 12 +- src/bin/pg_dump/pg_dump.c | 257 +++++++-------------------- src/bin/pg_dump/pg_dumpall.c | 69 ++----- src/bin/pg_dump/pg_restore.c | 167 +++++------------ 7 files changed, 350 insertions(+), 416 deletions(-) diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c index 41fb31b7253..d52f0c7f0bf 100644 --- a/src/bin/pg_dump/filter.c +++ b/src/bin/pg_dump/filter.c @@ -15,12 +15,27 @@ #include "common/logging.h" #include "common/string.h" #include "filter.h" +#include "pg_backup.h" +#include "pg_backup_utils.h" #include "lib/stringinfo.h" #include "pqexpbuffer.h" #define is_keyword_str(cstr, str, bytes) \ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0)) +/* + * State data for reading filter items from stream + */ +typedef struct +{ + FILE *fp; + const char *filename; + int lineno; + StringInfoData linebuff; + bool is_error; +} FilterStateData; + + /* * Following routines are called from pg_dump, pg_dumpall and pg_restore. * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is @@ -33,7 +48,7 @@ * Opens filter's file and initialize fstate structure. * Returns true on success. */ -bool +static bool filter_init(FilterStateData *fstate, const char *filename) { fstate->filename = filename; @@ -60,7 +75,7 @@ filter_init(FilterStateData *fstate, const char *filename) /* * Release allocated resources for the given filter. */ -void +static void filter_free(FilterStateData *fstate) { free(fstate->linebuff.data); @@ -118,7 +133,7 @@ filter_object_type_name(FilterObjectType fot) * This is mostly a convenience routine to avoid duplicating file closing code * in multiple callsites. */ -void +static void log_invalid_filter_format(FilterStateData *fstate, char *message) { if (fstate->fp != stdin) @@ -136,23 +151,26 @@ log_invalid_filter_format(FilterStateData *fstate, char *message) fstate->is_error = true; } + /* * Emit error message "The application doesn't support filter for object type ..." * * This is mostly a convenience routine to avoid duplicating file closing code * in multiple callsites. */ -void +static void log_unsupported_filter_object_type(FilterStateData *fstate, const char *appname, - FilterObjectType fot) + FilterObjectType fot, + bool is_include) { PQExpBuffer str = createPQExpBuffer(); printfPQExpBuffer(str, - "\"%s\" doesn't support filter for object type \"%s\".", + "\"%s\" doesn't support filter for object type \"%s\": %s", appname, - filter_object_type_name(fot)); + filter_object_type_name(fot), + is_include ? "include" : "exclude"); log_invalid_filter_format(fstate, str->data); } @@ -377,7 +395,7 @@ read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern) * error, the function will emit an appropriate error message before returning * false. */ -bool +static bool filter_read_item(FilterStateData *fstate, bool *is_include, char **objname, @@ -488,3 +506,142 @@ filter_read_item(FilterStateData *fstate, return false; } + +/* + * read_filters - retrieve object identifier patterns from file + * + * Parse the specified filter file for include and exclude patterns, and add + * them to the relevant lists. If the filename is "-" then filters will be + * read from STDIN rather than a file. + */ +void +read_filters(const char *appname, const char *filename, struct FilterOpts *filter_opts, struct type_opts *opts, bool *include_everything, int allow_include, int allow_exclude) +{ + FilterStateData fstate; + bool is_include; + char *objname; + FilterObjectType objtype; + + if (!filter_init(&fstate, filename)) + exit_nicely(1); // + + while (filter_read_item(&fstate, &is_include, &objname, &objtype)) + { + /* ignore comments and empty lines */ + if (objtype == FILTER_OBJECT_TYPE_NONE) + continue; + + /* allows "table and children" whenever table is allowed */ + if (allow_include & FILTER_OBJECT_TYPE_TABLE) + allow_include |= FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN; + if (allow_include & FILTER_OBJECT_TYPE_TABLE_DATA) + allow_include |= FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN; + if (allow_exclude & FILTER_OBJECT_TYPE_TABLE) + allow_exclude |= FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN; + if (allow_exclude & FILTER_OBJECT_TYPE_TABLE_DATA) + allow_exclude |= FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN; + + if (is_include && (allow_include & objtype) == 0) + log_unsupported_filter_object_type(&fstate, appname, objtype, is_include); + else if (!is_include && (allow_exclude & objtype) == 0) + log_unsupported_filter_object_type(&fstate, appname, objtype, is_include); + else if (objtype == FILTER_OBJECT_TYPE_DATABASE) + { + Assert(!is_include); + simple_string_list_append(&filter_opts->database_exclude_patterns, objname); + } + else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA) + { + // Assert(is_include); + if (!is_include) + log_invalid_filter_format(&fstate, + "\"exclude\" foreign data filter is not allowed"); + else + simple_string_list_append(&filter_opts->foreign_servers_include_patterns, + objname); + } + else if (objtype == FILTER_OBJECT_TYPE_FUNCTION) + { + Assert(is_include); + opts->selTypes = 1; + opts->selFunction = 1; + simple_string_list_append(&filter_opts->function_include_patterns, objname); + } + else if (objtype == FILTER_OBJECT_TYPE_INDEX) + { + Assert(is_include); + opts->selTypes = 1; + opts->selIndex = 1; + simple_string_list_append(&filter_opts->index_include_patterns, objname); + } + else if (objtype == FILTER_OBJECT_TYPE_SCHEMA) + { + if (is_include) + { + simple_string_list_append(&filter_opts->schema_include_patterns, objname); + *include_everything = false; + } + else + simple_string_list_append(&filter_opts->schema_exclude_patterns, objname); + } + else if (objtype == FILTER_OBJECT_TYPE_TABLE) + { + if (is_include) + { + opts->selTypes = 1; + opts->selTable = 1; + *include_everything = false; + simple_string_list_append(&filter_opts->table_include_patterns, objname); + } + else + simple_string_list_append(&filter_opts->table_exclude_patterns, objname); + } + else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA) + { + Assert(!is_include); + simple_string_list_append(&filter_opts->tabledata_exclude_patterns, + objname); + } + else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN) + { + Assert(!is_include); + simple_string_list_append(&filter_opts->tabledata_exclude_patterns_and_children, + objname); + } + else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN) + { + if (is_include) + { + simple_string_list_append(&filter_opts->table_include_patterns_and_children, objname); + *include_everything = false; + } + else + simple_string_list_append(&filter_opts->table_exclude_patterns_and_children, objname); + } + else if (objtype == FILTER_OBJECT_TYPE_TRIGGER) + { + Assert(is_include); + opts->selTypes = 1; + opts->selTrigger = 1; + simple_string_list_append(&filter_opts->trigger_include_patterns, objname); + } + else + { + // log_unsupported_filter_object_type(&fstate, "pg_dump", objtype); + // log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype); + // log_unsupported_filter_object_type(&fstate, "pg_restore", objtype); + pg_fatal("unhandled filter type: %d", objtype); + } + + if (objname) + free(objname); + + if (fstate.is_error) + exit_nicely(1); // + } + + if (fstate.is_error) + exit_nicely(1); // + + filter_free(&fstate); +} diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h index 28c5c9c8346..99a2a5a9a07 100644 --- a/src/bin/pg_dump/filter.h +++ b/src/bin/pg_dump/filter.h @@ -14,43 +14,56 @@ #define FILTER_H #include "lib/stringinfo.h" - -/* - * State data for reading filter items from stream - */ -typedef struct -{ - FILE *fp; - const char *filename; - int lineno; - StringInfoData linebuff; - bool is_error; -} FilterStateData; +#include "fe_utils/simple_list.h" /* * List of objects that can be specified in filter file */ typedef enum { - FILTER_OBJECT_TYPE_NONE, - FILTER_OBJECT_TYPE_TABLE_DATA, - FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN, - FILTER_OBJECT_TYPE_DATABASE, - FILTER_OBJECT_TYPE_FOREIGN_DATA, - FILTER_OBJECT_TYPE_FUNCTION, - FILTER_OBJECT_TYPE_INDEX, - FILTER_OBJECT_TYPE_SCHEMA, - FILTER_OBJECT_TYPE_TABLE, - FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN, - FILTER_OBJECT_TYPE_TRIGGER + FILTER_OBJECT_TYPE_NONE = 1<<0, + FILTER_OBJECT_TYPE_TABLE_DATA = 1<<1, + FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN = 1<<2, + FILTER_OBJECT_TYPE_DATABASE = 1<<3, + FILTER_OBJECT_TYPE_FOREIGN_DATA = 1<<4, + FILTER_OBJECT_TYPE_FUNCTION = 1<<5, + FILTER_OBJECT_TYPE_INDEX = 1<<6, + FILTER_OBJECT_TYPE_SCHEMA = 1<<7, + FILTER_OBJECT_TYPE_TABLE = 1<<8, + FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN = 1<<9, + FILTER_OBJECT_TYPE_TRIGGER = 1<<10, } FilterObjectType; -extern bool filter_init(FilterStateData *fstate, const char *filename); -extern void filter_free(FilterStateData *fstate); +struct FilterOpts { + SimpleStringList schema_include_patterns; + SimpleOidList schema_include_oids; + SimpleStringList schema_exclude_patterns; + SimpleOidList schema_exclude_oids; + + SimpleStringList table_include_patterns; + SimpleStringList table_include_patterns_and_children; + SimpleOidList table_include_oids; + SimpleStringList table_exclude_patterns; + SimpleStringList table_exclude_patterns_and_children; + SimpleOidList table_exclude_oids; + SimpleStringList tabledata_exclude_patterns; + SimpleStringList tabledata_exclude_patterns_and_children; + SimpleOidList tabledata_exclude_oids; + SimpleStringList foreign_servers_include_patterns; + SimpleOidList foreign_servers_include_oids; + + SimpleStringList extension_include_patterns; + SimpleOidList extension_include_oids; + + SimpleStringList database_exclude_patterns; + SimpleStringList index_include_patterns; + SimpleStringList function_include_patterns; + SimpleStringList trigger_include_patterns; +}; -extern void log_invalid_filter_format(FilterStateData *fstate, char *message); -extern void log_unsupported_filter_object_type(FilterStateData *fstate, - const char *appname, FilterObjectType fot); -extern bool filter_read_item(FilterStateData *fstate, bool *is_include, char **objname, FilterObjectType *objtype); +struct type_opts; /* fwd decl */ +extern void read_filters(const char *appname, const char *filename, struct FilterOpts *filteropts, + struct type_opts *opts, bool *include_everything, + int allow_include, int allow_exclude); #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index aba780ef4b1..0794ec3a301 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -26,6 +26,7 @@ #include "common/compression.h" #include "fe_utils/simple_list.h" #include "libpq-fe.h" +#include "filter.h" typedef enum trivalue @@ -90,6 +91,15 @@ typedef struct _connParams char *override_dbname; } ConnParams; +struct type_opts +{ + int selTypes; + int selIndex; + int selFunction; + int selTrigger; + int selTable; +}; + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -127,11 +137,8 @@ typedef struct _restoreOptions int format; char *formatName; - int selTypes; - int selIndex; - int selFunction; - int selTrigger; - int selTable; + struct type_opts typeopts; + SimpleStringList indexNames; SimpleStringList functionNames; SimpleStringList schemaNames; diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 3337d34e405..57e73bd608d 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -2890,7 +2890,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) } else if (ropt->schemaNames.head != NULL || ropt->schemaExcludeNames.head != NULL || - ropt->selTypes) + ropt->typeopts.selTypes) { /* * In a selective dump/restore, we want to restore these dependent @@ -2930,7 +2930,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) simple_string_list_member(&ropt->schemaExcludeNames, te->namespace)) return 0; - if (ropt->selTypes) + if (ropt->typeopts.selTypes) { if (strcmp(te->desc, "TABLE") == 0 || strcmp(te->desc, "TABLE DATA") == 0 || @@ -2941,7 +2941,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) strcmp(te->desc, "SEQUENCE") == 0 || strcmp(te->desc, "SEQUENCE SET") == 0) { - if (!ropt->selTable) + if (!ropt->typeopts.selTable) return 0; if (ropt->tableNames.head != NULL && !simple_string_list_member(&ropt->tableNames, te->tag)) @@ -2949,7 +2949,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) } else if (strcmp(te->desc, "INDEX") == 0) { - if (!ropt->selIndex) + if (!ropt->typeopts.selIndex) return 0; if (ropt->indexNames.head != NULL && !simple_string_list_member(&ropt->indexNames, te->tag)) @@ -2959,7 +2959,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) strcmp(te->desc, "AGGREGATE") == 0 || strcmp(te->desc, "PROCEDURE") == 0) { - if (!ropt->selFunction) + if (!ropt->typeopts.selFunction) return 0; if (ropt->functionNames.head != NULL && !simple_string_list_member(&ropt->functionNames, te->tag)) @@ -2967,7 +2967,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH) } else if (strcmp(te->desc, "TRIGGER") == 0) { - if (!ropt->selTrigger) + if (!ropt->typeopts.selTrigger) return 0; if (ropt->triggerNames.head != NULL && !simple_string_list_member(&ropt->triggerNames, te->tag)) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 42b8f8d91d5..b4f1ccb2f9b 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -114,26 +114,7 @@ static pg_compress_algorithm compression_algorithm = PG_COMPRESSION_NONE; * The string lists record the patterns given by command-line switches, * which we then convert to lists of OIDs of matching objects. */ -static SimpleStringList schema_include_patterns = {NULL, NULL}; -static SimpleOidList schema_include_oids = {NULL, NULL}; -static SimpleStringList schema_exclude_patterns = {NULL, NULL}; -static SimpleOidList schema_exclude_oids = {NULL, NULL}; - -static SimpleStringList table_include_patterns = {NULL, NULL}; -static SimpleStringList table_include_patterns_and_children = {NULL, NULL}; -static SimpleOidList table_include_oids = {NULL, NULL}; -static SimpleStringList table_exclude_patterns = {NULL, NULL}; -static SimpleStringList table_exclude_patterns_and_children = {NULL, NULL}; -static SimpleOidList table_exclude_oids = {NULL, NULL}; -static SimpleStringList tabledata_exclude_patterns = {NULL, NULL}; -static SimpleStringList tabledata_exclude_patterns_and_children = {NULL, NULL}; -static SimpleOidList tabledata_exclude_oids = {NULL, NULL}; - -static SimpleStringList foreign_servers_include_patterns = {NULL, NULL}; -static SimpleOidList foreign_servers_include_oids = {NULL, NULL}; - -static SimpleStringList extension_include_patterns = {NULL, NULL}; -static SimpleOidList extension_include_oids = {NULL, NULL}; +struct FilterOpts filter_opts; static const CatalogId nilCatalogId = {0, 0}; @@ -327,7 +308,6 @@ static char *get_synchronized_snapshot(Archive *fout); static void setupDumpWorker(Archive *AH); static TableInfo *getRootTableInfo(const TableInfo *tbinfo); static bool forcePartitionRootLoad(const TableInfo *tbinfo); -static void read_dump_filters(const char *filename, DumpOptions *dopt); int @@ -495,7 +475,7 @@ main(int argc, char **argv) break; case 'e': /* include extension(s) */ - simple_string_list_append(&extension_include_patterns, optarg); + simple_string_list_append(&filter_opts.extension_include_patterns, optarg); dopt.include_everything = false; break; @@ -523,12 +503,12 @@ main(int argc, char **argv) break; case 'n': /* include schema(s) */ - simple_string_list_append(&schema_include_patterns, optarg); + simple_string_list_append(&filter_opts.schema_include_patterns, optarg); dopt.include_everything = false; break; case 'N': /* exclude schema(s) */ - simple_string_list_append(&schema_exclude_patterns, optarg); + simple_string_list_append(&filter_opts.schema_exclude_patterns, optarg); break; case 'O': /* Don't reconnect to match owner */ @@ -552,12 +532,12 @@ main(int argc, char **argv) break; case 't': /* include table(s) */ - simple_string_list_append(&table_include_patterns, optarg); + simple_string_list_append(&filter_opts.table_include_patterns, optarg); dopt.include_everything = false; break; case 'T': /* exclude table(s) */ - simple_string_list_append(&table_exclude_patterns, optarg); + simple_string_list_append(&filter_opts.table_exclude_patterns, optarg); break; case 'U': @@ -600,7 +580,7 @@ main(int argc, char **argv) break; case 4: /* exclude table(s) data */ - simple_string_list_append(&tabledata_exclude_patterns, optarg); + simple_string_list_append(&filter_opts.tabledata_exclude_patterns, optarg); break; case 5: /* section */ @@ -639,28 +619,35 @@ main(int argc, char **argv) break; case 11: /* include foreign data */ - simple_string_list_append(&foreign_servers_include_patterns, + simple_string_list_append(&filter_opts.foreign_servers_include_patterns, optarg); break; case 12: /* include table(s) and their children */ - simple_string_list_append(&table_include_patterns_and_children, + simple_string_list_append(&filter_opts.table_include_patterns_and_children, optarg); dopt.include_everything = false; break; case 13: /* exclude table(s) and their children */ - simple_string_list_append(&table_exclude_patterns_and_children, + simple_string_list_append(&filter_opts.table_exclude_patterns_and_children, optarg); break; case 14: /* exclude data of table(s) and children */ - simple_string_list_append(&tabledata_exclude_patterns_and_children, + simple_string_list_append(&filter_opts.tabledata_exclude_patterns_and_children, optarg); break; case 15: /* object filters from file */ - read_dump_filters(optarg, &dopt); + { + struct type_opts typeopts; + // XXX: do something with it.... + read_filters(progname, optarg, &filter_opts, &typeopts, &dopt.include_everything, + FILTER_OBJECT_TYPE_SCHEMA | FILTER_OBJECT_TYPE_TABLE | FILTER_OBJECT_TYPE_FOREIGN_DATA, /* inclusions */ + FILTER_OBJECT_TYPE_SCHEMA | FILTER_OBJECT_TYPE_TABLE | FILTER_OBJECT_TYPE_TABLE_DATA | /*XXX not really:*/FILTER_OBJECT_TYPE_FOREIGN_DATA /* exclusions */ + ); + } break; default: @@ -701,10 +688,10 @@ main(int argc, char **argv) if (dopt.dataOnly && dopt.schemaOnly) pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together"); - if (dopt.schemaOnly && foreign_servers_include_patterns.head != NULL) + if (dopt.schemaOnly && filter_opts.foreign_servers_include_patterns.head != NULL) pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together"); - if (numWorkers > 1 && foreign_servers_include_patterns.head != NULL) + if (numWorkers > 1 && filter_opts.foreign_servers_include_patterns.head != NULL) pg_fatal("option --include-foreign-data is not supported with parallel backup"); if (dopt.dataOnly && dopt.outputClean) @@ -828,57 +815,59 @@ main(int argc, char **argv) pg_log_info("last built-in OID is %u", g_last_builtin_oid); /* Expand schema selection patterns into OID lists */ - if (schema_include_patterns.head != NULL) + if (filter_opts.schema_include_patterns.head != NULL) { - expand_schema_name_patterns(fout, &schema_include_patterns, - &schema_include_oids, + expand_schema_name_patterns(fout, &filter_opts.schema_include_patterns, + &filter_opts.schema_include_oids, strict_names); - if (schema_include_oids.head == NULL) + if (filter_opts.schema_include_oids.head == NULL) pg_fatal("no matching schemas were found"); } - expand_schema_name_patterns(fout, &schema_exclude_patterns, - &schema_exclude_oids, + expand_schema_name_patterns(fout, &filter_opts.schema_exclude_patterns, + &filter_opts.schema_exclude_oids, false); /* non-matching exclusion patterns aren't an error */ /* Expand table selection patterns into OID lists */ - expand_table_name_patterns(fout, &table_include_patterns, - &table_include_oids, + expand_table_name_patterns(fout, &filter_opts.table_include_patterns, + &filter_opts.table_include_oids, strict_names, false); - expand_table_name_patterns(fout, &table_include_patterns_and_children, - &table_include_oids, + + expand_table_name_patterns(fout, &filter_opts.table_include_patterns_and_children, + &filter_opts.table_include_oids, strict_names, true); - if ((table_include_patterns.head != NULL || - table_include_patterns_and_children.head != NULL) && - table_include_oids.head == NULL) + + if ((filter_opts.table_include_patterns.head != NULL || + filter_opts.table_include_patterns_and_children.head != NULL) && + filter_opts.table_include_oids.head == NULL) pg_fatal("no matching tables were found"); - expand_table_name_patterns(fout, &table_exclude_patterns, - &table_exclude_oids, + expand_table_name_patterns(fout, &filter_opts.table_exclude_patterns, + &filter_opts.table_exclude_oids, false, false); - expand_table_name_patterns(fout, &table_exclude_patterns_and_children, - &table_exclude_oids, + expand_table_name_patterns(fout, &filter_opts.table_exclude_patterns_and_children, + &filter_opts.table_exclude_oids, false, true); - expand_table_name_patterns(fout, &tabledata_exclude_patterns, - &tabledata_exclude_oids, + expand_table_name_patterns(fout, &filter_opts.tabledata_exclude_patterns, + &filter_opts.tabledata_exclude_oids, false, false); - expand_table_name_patterns(fout, &tabledata_exclude_patterns_and_children, - &tabledata_exclude_oids, + expand_table_name_patterns(fout, &filter_opts.tabledata_exclude_patterns_and_children, + &filter_opts.tabledata_exclude_oids, false, true); - expand_foreign_server_name_patterns(fout, &foreign_servers_include_patterns, - &foreign_servers_include_oids); + expand_foreign_server_name_patterns(fout, &filter_opts.foreign_servers_include_patterns, + &filter_opts.foreign_servers_include_oids); /* non-matching exclusion patterns aren't an error */ /* Expand extension selection patterns into OID lists */ - if (extension_include_patterns.head != NULL) + if (filter_opts.extension_include_patterns.head != NULL) { - expand_extension_name_patterns(fout, &extension_include_patterns, - &extension_include_oids, + expand_extension_name_patterns(fout, &filter_opts.extension_include_patterns, + &filter_opts.extension_include_oids, strict_names); - if (extension_include_oids.head == NULL) + if (filter_opts.extension_include_oids.head == NULL) pg_fatal("no matching extensions were found"); } @@ -1729,11 +1718,11 @@ selectDumpableNamespace(NamespaceInfo *nsinfo, Archive *fout) * namespaces. If specific namespaces are being dumped, dump just those * namespaces. Otherwise, dump all non-system namespaces. */ - if (table_include_oids.head != NULL) + if (filter_opts.table_include_oids.head != NULL) nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_NONE; - else if (schema_include_oids.head != NULL) + else if (filter_opts.schema_include_oids.head != NULL) nsinfo->dobj.dump_contains = nsinfo->dobj.dump = - simple_oid_list_member(&schema_include_oids, + simple_oid_list_member(&filter_opts.schema_include_oids, nsinfo->dobj.catId.oid) ? DUMP_COMPONENT_ALL : DUMP_COMPONENT_NONE; else if (fout->remoteVersion >= 90600 && @@ -1782,7 +1771,7 @@ selectDumpableNamespace(NamespaceInfo *nsinfo, Archive *fout) * In any case, a namespace can be excluded by an exclusion switch */ if (nsinfo->dobj.dump_contains && - simple_oid_list_member(&schema_exclude_oids, + simple_oid_list_member(&filter_opts.schema_exclude_oids, nsinfo->dobj.catId.oid)) nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_NONE; @@ -1810,8 +1799,8 @@ selectDumpableTable(TableInfo *tbinfo, Archive *fout) * If specific tables are being dumped, dump just those tables; else, dump * according to the parent namespace's dump flag. */ - if (table_include_oids.head != NULL) - tbinfo->dobj.dump = simple_oid_list_member(&table_include_oids, + if (filter_opts.table_include_oids.head != NULL) + tbinfo->dobj.dump = simple_oid_list_member(&filter_opts.table_include_oids, tbinfo->dobj.catId.oid) ? DUMP_COMPONENT_ALL : DUMP_COMPONENT_NONE; else @@ -1821,7 +1810,7 @@ selectDumpableTable(TableInfo *tbinfo, Archive *fout) * In any case, a table can be excluded by an exclusion switch */ if (tbinfo->dobj.dump && - simple_oid_list_member(&table_exclude_oids, + simple_oid_list_member(&filter_opts.table_exclude_oids, tbinfo->dobj.catId.oid)) tbinfo->dobj.dump = DUMP_COMPONENT_NONE; } @@ -2005,9 +1994,9 @@ selectDumpableExtension(ExtensionInfo *extinfo, DumpOptions *dopt) else { /* check if there is a list of extensions to dump */ - if (extension_include_oids.head != NULL) + if (filter_opts.extension_include_oids.head != NULL) extinfo->dobj.dump = extinfo->dobj.dump_contains = - simple_oid_list_member(&extension_include_oids, + simple_oid_list_member(&filter_opts.extension_include_oids, extinfo->dobj.catId.oid) ? DUMP_COMPONENT_ALL : DUMP_COMPONENT_NONE; else @@ -2719,8 +2708,8 @@ makeTableDataInfo(DumpOptions *dopt, TableInfo *tbinfo) return; /* Skip FOREIGN TABLEs (no data to dump) unless requested explicitly */ if (tbinfo->relkind == RELKIND_FOREIGN_TABLE && - (foreign_servers_include_oids.head == NULL || - !simple_oid_list_member(&foreign_servers_include_oids, + (filter_opts.foreign_servers_include_oids.head == NULL || + !simple_oid_list_member(&filter_opts.foreign_servers_include_oids, tbinfo->foreign_server))) return; /* Skip partitioned tables (data in partitions) */ @@ -2733,7 +2722,7 @@ makeTableDataInfo(DumpOptions *dopt, TableInfo *tbinfo) return; /* Check that the data is not explicitly excluded */ - if (simple_oid_list_member(&tabledata_exclude_oids, + if (simple_oid_list_member(&filter_opts.tabledata_exclude_oids, tbinfo->dobj.catId.oid)) return; @@ -17849,8 +17838,8 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[], * Check if this extension is listed as to include in the dump. If * not, any table data associated with it is discarded. */ - if (extension_include_oids.head != NULL && - !simple_oid_list_member(&extension_include_oids, + if (filter_opts.extension_include_oids.head != NULL && + !simple_oid_list_member(&filter_opts.extension_include_oids, curext->dobj.catId.oid)) continue; @@ -17883,8 +17872,8 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[], if (!(curext->dobj.dump & DUMP_COMPONENT_DEFINITION)) { /* check table explicitly requested */ - if (table_include_oids.head != NULL && - simple_oid_list_member(&table_include_oids, + if (filter_opts.table_include_oids.head != NULL && + simple_oid_list_member(&filter_opts.table_include_oids, configtbloid)) dumpobj = true; @@ -17895,13 +17884,13 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[], } /* check table excluded by an exclusion switch */ - if (table_exclude_oids.head != NULL && - simple_oid_list_member(&table_exclude_oids, + if (filter_opts.table_exclude_oids.head != NULL && + simple_oid_list_member(&filter_opts.table_exclude_oids, configtbloid)) dumpobj = false; /* check schema excluded by an exclusion switch */ - if (simple_oid_list_member(&schema_exclude_oids, + if (simple_oid_list_member(&filter_opts.schema_exclude_oids, configtbl->dobj.namespace->dobj.catId.oid)) dumpobj = false; @@ -18484,111 +18473,3 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, if (!res) pg_log_warning("could not parse %s array", "reloptions"); } - -/* - * read_dump_filters - retrieve object identifier patterns from file - * - * Parse the specified filter file for include and exclude patterns, and add - * them to the relevant lists. If the filename is "-" then filters will be - * read from STDIN rather than a file. - */ -static void -read_dump_filters(const char *filename, DumpOptions *dopt) -{ - FilterStateData fstate; - bool is_include; - char *objname; - FilterObjectType objtype; - - if (!filter_init(&fstate, filename)) - exit_nicely(1); - - while (filter_read_item(&fstate, &is_include, &objname, &objtype)) - { - /* ignore comments and empty lines */ - if (objtype == FILTER_OBJECT_TYPE_NONE) - continue; - - if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA) - { - if (is_include) - { - log_invalid_filter_format(&fstate, - "\"include\" table data filter is not allowed"); - break; - } - else - simple_string_list_append(&tabledata_exclude_patterns, - objname); - } - else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN) - { - if (is_include) - { - log_invalid_filter_format(&fstate, - "\"include\" table data and children filter is not allowed"); - break; - } - else - simple_string_list_append(&tabledata_exclude_patterns_and_children, - objname); - } - else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA) - { - if (is_include) - simple_string_list_append(&foreign_servers_include_patterns, - objname); - else - { - log_invalid_filter_format(&fstate, - "\"exclude\" foreign data filter is not allowed"); - break; - } - } - else if (objtype == FILTER_OBJECT_TYPE_SCHEMA) - { - if (is_include) - { - simple_string_list_append(&schema_include_patterns, - objname); - dopt->include_everything = false; - } - else - simple_string_list_append(&schema_exclude_patterns, - objname); - } - else if (objtype == FILTER_OBJECT_TYPE_TABLE) - { - if (is_include) - { - simple_string_list_append(&table_include_patterns, objname); - dopt->include_everything = false; - } - else - simple_string_list_append(&table_exclude_patterns, objname); - } - else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN) - { - if (is_include) - { - simple_string_list_append(&table_include_patterns_and_children, objname); - dopt->include_everything = false; - } - else - simple_string_list_append(&table_exclude_patterns_and_children, objname); - } - else - { - log_unsupported_filter_object_type(&fstate, "pg_dump", objtype); - break; - } - - if (objname) - free(objname); - } - - filter_free(&fstate); - - if (fstate.is_error) - exit_nicely(1); -} diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index d58eaaaa063..502ab128337 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -82,7 +82,6 @@ static PGresult *executeQuery(PGconn *conn, const char *query); static void executeCommand(PGconn *conn, const char *query); static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, SimpleStringList *names); -static void read_dumpall_filters(const char *filename, SimpleStringList *patterns); static char pg_dump_bin[MAXPGPATH]; static const char *progname; @@ -120,8 +119,8 @@ static char role_catalog[10]; static FILE *OPF; static char *filename = NULL; -static SimpleStringList database_exclude_patterns = {NULL, NULL}; static SimpleStringList database_exclude_names = {NULL, NULL}; +struct FilterOpts filter_opts; #define exit_nicely(code) exit(code) @@ -355,7 +354,7 @@ main(int argc, char *argv[]) break; case 6: - simple_string_list_append(&database_exclude_patterns, optarg); + simple_string_list_append(&filter_opts.database_exclude_patterns, optarg); break; case 7: @@ -364,7 +363,12 @@ main(int argc, char *argv[]) break; case 8: - read_dumpall_filters(optarg, &database_exclude_patterns); + { + bool foo; + struct type_opts typeopts; + read_filters(progname, optarg, &filter_opts, &typeopts, &foo, + 0, FILTER_OBJECT_TYPE_DATABASE); + } break; default: @@ -383,7 +387,8 @@ main(int argc, char *argv[]) exit_nicely(1); } - if (database_exclude_patterns.head != NULL && + if (filter_opts.database_exclude_patterns.head != NULL && + (globals_only || roles_only || tablespaces_only)) { pg_log_error("option --exclude-database cannot be used together with -g/--globals-only, -r/--roles-only, or -t/--tablespaces-only"); @@ -495,7 +500,7 @@ main(int argc, char *argv[]) /* * Get a list of database names that match the exclude patterns */ - expand_dbname_patterns(conn, &database_exclude_patterns, + expand_dbname_patterns(conn, &filter_opts.database_exclude_patterns, &database_exclude_names); /* @@ -1939,55 +1944,3 @@ hash_string_pointer(char *s) return hash_bytes(ss, strlen(s)); } - -/* - * read_dumpall_filters - retrieve database identifier patterns from file - * - * Parse the specified filter file for include and exclude patterns, and add - * them to the relevant lists. If the filename is "-" then filters will be - * read from STDIN rather than a file. - * - * At the moment, the only allowed filter is for database exclusion. - */ -static void -read_dumpall_filters(const char *filename, SimpleStringList *pattern) -{ - FilterStateData fstate; - bool is_include; - char *objname; - FilterObjectType objtype; - - if (!filter_init(&fstate, filename)) - exit_nicely(1); - - while (filter_read_item(&fstate, &is_include, &objname, &objtype)) - { - if (objtype == FILTER_OBJECT_TYPE_NONE) - continue; - - if (objtype == FILTER_OBJECT_TYPE_DATABASE) - { - if (!is_include) - simple_string_list_append(pattern, objname); - else - { - log_invalid_filter_format(&fstate, - "\"include\" database filter is not allowed"); - break; - } - } - else - { - log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype); - break; - } - - if (objname) - free(objname); - } - - filter_free(&fstate); - - if (fstate.is_error) - exit_nicely(1); -} diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 0ea71351737..02390886946 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -53,12 +53,12 @@ #include "pg_backup_utils.h" static void usage(const char *progname); -static void read_restore_filters(const char *filename, RestoreOptions *dopt); int main(int argc, char **argv) { RestoreOptions *opts; + struct FilterOpts filter_opts = {0}; int c; int exit_code; int numWorkers = 1; @@ -202,11 +202,11 @@ main(int argc, char **argv) break; case 'n': /* Dump data for this schema only */ - simple_string_list_append(&opts->schemaNames, optarg); + simple_string_list_append(&filter_opts.schema_include_patterns, optarg); break; case 'N': /* Do not dump data for this schema */ - simple_string_list_append(&opts->schemaExcludeNames, optarg); + simple_string_list_append(&filter_opts.schema_exclude_patterns, optarg); break; case 'O': @@ -221,19 +221,19 @@ main(int argc, char **argv) /* no-op, still accepted for backwards compatibility */ break; case 'P': /* Function */ - opts->selTypes = 1; - opts->selFunction = 1; - simple_string_list_append(&opts->functionNames, optarg); + opts->typeopts.selTypes = 1; + opts->typeopts.selFunction = 1; + simple_string_list_append(&filter_opts.function_include_patterns, optarg); break; case 'I': /* Index */ - opts->selTypes = 1; - opts->selIndex = 1; - simple_string_list_append(&opts->indexNames, optarg); + opts->typeopts.selTypes = 1; + opts->typeopts.selIndex = 1; + simple_string_list_append(&filter_opts.index_include_patterns, optarg); break; case 'T': /* Trigger */ - opts->selTypes = 1; - opts->selTrigger = 1; - simple_string_list_append(&opts->triggerNames, optarg); + opts->typeopts.selTypes = 1; + opts->typeopts.selTrigger = 1; + simple_string_list_append(&filter_opts.trigger_include_patterns, optarg); break; case 's': /* dump schema only */ opts->schemaOnly = 1; @@ -243,9 +243,9 @@ main(int argc, char **argv) opts->superuser = pg_strdup(optarg); break; case 't': /* Dump specified table(s) only */ - opts->selTypes = 1; - opts->selTable = 1; - simple_string_list_append(&opts->tableNames, optarg); + opts->typeopts.selTypes = 1; + opts->typeopts.selTable = 1; + simple_string_list_append(&filter_opts.table_include_patterns, optarg); break; case 'U': @@ -290,7 +290,14 @@ main(int argc, char **argv) break; case 4: - read_restore_filters(optarg, opts); + { + bool include_everything = opts->include_everything; + read_filters(progname, optarg, &filter_opts, &opts->typeopts, &include_everything, + FILTER_OBJECT_TYPE_SCHEMA | FILTER_OBJECT_TYPE_TABLE_DATA | FILTER_OBJECT_TYPE_FOREIGN_DATA | FILTER_OBJECT_TYPE_FUNCTION | FILTER_OBJECT_TYPE_INDEX | FILTER_OBJECT_TYPE_TABLE | FILTER_OBJECT_TYPE_TRIGGER, /* allowed to include */ + FILTER_OBJECT_TYPE_SCHEMA /* allowed to exclude */ + ); + opts->include_everything = include_everything; + } break; default: @@ -300,6 +307,28 @@ main(int argc, char **argv) } } + /* + * Any values read into filter_opts are appended to the corresponding + * values in opts, which are used during restore. + */ + for (SimpleStringListCell *cell = filter_opts.trigger_include_patterns.head; cell; cell = cell->next) + simple_string_list_append(&opts->triggerNames, cell->val); + + for (SimpleStringListCell *cell = filter_opts.schema_include_patterns.head; cell; cell = cell->next) + simple_string_list_append(&opts->schemaNames, cell->val); + + for (SimpleStringListCell *cell = filter_opts.schema_exclude_patterns.head; cell; cell = cell->next) + simple_string_list_append(&opts->schemaExcludeNames, cell->val); + + for (SimpleStringListCell *cell = filter_opts.function_include_patterns.head; cell; cell = cell->next) + simple_string_list_append(&opts->functionNames, cell->val); + + for (SimpleStringListCell *cell = filter_opts.index_include_patterns.head; cell; cell = cell->next) + simple_string_list_append(&opts->indexNames, cell->val); + + for (SimpleStringListCell *cell = filter_opts.table_include_patterns.head; cell; cell = cell->next) + simple_string_list_append(&opts->tableNames, cell->val); + /* Get file name from command line */ if (optind < argc) inputFileSpec = argv[optind++]; @@ -502,109 +531,3 @@ usage(const char *progname) printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT); printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); } - -/* - * read_restore_filters - retrieve object identifier patterns from file - * - * Parse the specified filter file for include and exclude patterns, and add - * them to the relevant lists. If the filename is "-" then filters will be - * read from STDIN rather than a file. - */ -static void -read_restore_filters(const char *filename, RestoreOptions *opts) -{ - FilterStateData fstate; - bool is_include; - char *objname; - FilterObjectType objtype; - - if (!filter_init(&fstate, filename)) - exit_nicely(1); - - while (filter_read_item(&fstate, &is_include, &objname, &objtype)) - { - /* ignore comments or empty lines */ - if (objtype == FILTER_OBJECT_TYPE_NONE) - continue; - - if (objtype == FILTER_OBJECT_TYPE_FUNCTION) - { - if (is_include) - { - opts->selTypes = 1; - opts->selFunction = 1; - simple_string_list_append(&opts->functionNames, objname); - } - else - { - log_invalid_filter_format(&fstate, - "\"exclude\" function filter is not allowed"); - break; - } - } - else if (objtype == FILTER_OBJECT_TYPE_INDEX) - { - if (is_include) - { - opts->selTypes = 1; - opts->selIndex = 1; - simple_string_list_append(&opts->indexNames, objname); - } - else - { - log_invalid_filter_format(&fstate, - "\"exclude\" index filter is not allowed"); - break; - } - } - else if (objtype == FILTER_OBJECT_TYPE_SCHEMA) - { - if (is_include) - simple_string_list_append(&opts->schemaNames, objname); - else - simple_string_list_append(&opts->schemaExcludeNames, objname); - } - else if (objtype == FILTER_OBJECT_TYPE_TABLE) - { - if (is_include) - { - opts->selTypes = 1; - opts->selTable = 1; - simple_string_list_append(&opts->tableNames, objname); - } - else - { - log_invalid_filter_format(&fstate, - "\"exclude\" table filter is not allowed"); - break; - } - } - else if (objtype == FILTER_OBJECT_TYPE_TRIGGER) - { - if (is_include) - { - opts->selTypes = 1; - opts->selTrigger = 1; - simple_string_list_append(&opts->triggerNames, objname); - } - else - { - log_invalid_filter_format(&fstate, - "\"exclude\" trigger filter is not allowed"); - break; - } - } - else - { - log_unsupported_filter_object_type(&fstate, "pg_restore", objtype); - break; - } - - if (objname) - free(objname); - } - - filter_free(&fstate); - if (fstate.is_error) - exit_nicely(1); -} -- 2.34.1