From 177fabe331cbf21b695c9d8f7bab513d14e8f727 Mon Sep 17 00:00:00 2001 From: Matthias van de Meent Date: Thu, 7 Jan 2021 16:39:57 +0100 Subject: [PATCH v2 1/5] Add progress reporting for excluded rows. COPY ... FROM ... WHERE (condition) will exclude rows that do not match the condition, which we now register as explicitly being excluded from insertion. --- doc/src/sgml/monitoring.sgml | 10 ++++++++++ src/backend/catalog/system_views.sql | 3 ++- src/backend/commands/copyfrom.c | 5 +++++ src/include/commands/progress.h | 1 + src/test/regress/expected/rules.out | 3 ++- 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index c602ee4427..dc998bc5f7 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -6571,6 +6571,16 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, Number of lines already processed by COPY command. + + + + lines_excluded bigint + + + Number of lines not processed because they were excluded by the + WHERE clause of the COPY command. + + diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index fa58afd9d7..a9bd983419 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1131,7 +1131,8 @@ CREATE VIEW pg_stat_progress_copy AS S.relid AS relid, S.param1 AS bytes_processed, S.param2 AS bytes_total, - S.param3 AS lines_processed + S.param3 AS lines_processed, + S.param4 AS lines_excluded FROM pg_stat_get_progress_info('COPY') AS S LEFT JOIN pg_database D ON S.datid = D.oid; diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index c39cc736ed..86e6a422e2 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -540,6 +540,7 @@ CopyFrom(CopyFromState cstate) CopyInsertMethod insertMethod; CopyMultiInsertInfo multiInsertInfo = {0}; /* pacify compiler */ uint64 processed = 0; + uint64 excluded = 0; bool has_before_insert_row_trig; bool has_instead_insert_row_trig; bool leafpart_use_multi_insert = false; @@ -868,7 +869,11 @@ CopyFrom(CopyFromState cstate) econtext->ecxt_scantuple = myslot; /* Skip items that don't match COPY's WHERE clause */ if (!ExecQual(cstate->qualexpr, econtext)) + { + /* Report that this tuple was filtered out by the WHERE clause */ + pgstat_progress_update_param(PROGRESS_COPY_LINES_EXCLUDED, ++excluded); continue; + } } /* Determine the partition to insert the tuple into */ diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index 95ec5d02e9..e6f6545033 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -137,5 +137,6 @@ #define PROGRESS_COPY_BYTES_PROCESSED 0 #define PROGRESS_COPY_BYTES_TOTAL 1 #define PROGRESS_COPY_LINES_PROCESSED 2 +#define PROGRESS_COPY_LINES_EXCLUDED 3 #endif diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 6173473de9..25a863627e 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1950,7 +1950,8 @@ pg_stat_progress_copy| SELECT s.pid, s.relid, s.param1 AS bytes_processed, s.param2 AS bytes_total, - s.param3 AS lines_processed + s.param3 AS lines_processed, + s.param4 AS lines_excluded FROM (pg_stat_get_progress_info('COPY'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_create_index| SELECT s.pid, -- 2.20.1