From 637b83b5deaa2a336d54601dff9ed3fc09f45061 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: Mon, 24 Aug 2020 13:45:50 +0500 Subject: [PATCH 2/2] Pull statistic for a foreign table from remote server. Add the postgres_fdw_stat() routine that convert statistics on the relation into json format. New fdw API routine GetForeignRelStat() implements access to this machinery and returns JSON string to the caller. This function is called by ANALYZE command as an attempt to reduce the cost of updating statistics. Add foreign relation analyze into the VACUUM ANALYZE and autovacuum. --- contrib/postgres_fdw/deparse.c | 8 + .../postgres_fdw/expected/foreign_stat.out | 115 +++- .../postgres_fdw/postgres_fdw--1.0--1.1.sql | 2 +- contrib/postgres_fdw/postgres_fdw.c | 53 +- contrib/postgres_fdw/postgres_fdw.h | 1 + contrib/postgres_fdw/sql/foreign_stat.sql | 41 +- src/backend/commands/analyze.c | 580 ++++++++++++++++++ src/backend/commands/vacuum.c | 13 +- src/backend/utils/adt/json.c | 6 + src/include/commands/vacuum.h | 1 + src/include/foreign/fdwapi.h | 2 + src/include/utils/json.h | 1 + 12 files changed, 798 insertions(+), 25 deletions(-) diff --git a/contrib/postgres_fdw/deparse.c b/contrib/postgres_fdw/deparse.c index ad37a74221..9514fe996a 100644 --- a/contrib/postgres_fdw/deparse.c +++ b/contrib/postgres_fdw/deparse.c @@ -2053,6 +2053,14 @@ deparseAnalyzeSizeSql(StringInfo buf, Relation rel) appendStringInfo(buf, "::pg_catalog.regclass) / %d", BLCKSZ); } +void +deparseGetStatSql(StringInfo buf, Relation rel) +{ + appendStringInfo(buf, "SELECT * FROM public.postgres_fdw_stat('"); + deparseRelation(buf, rel); + appendStringInfoString(buf, "');"); +} + /* * Construct SELECT statement to acquire sample rows of given relation. * diff --git a/contrib/postgres_fdw/expected/foreign_stat.out b/contrib/postgres_fdw/expected/foreign_stat.out index 28a470bccc..a5d5ec8564 100644 --- a/contrib/postgres_fdw/expected/foreign_stat.out +++ b/contrib/postgres_fdw/expected/foreign_stat.out @@ -1,18 +1,107 @@ CREATE TABLE ltable (a int, b real); CREATE FOREIGN TABLE ftable (a int) server loopback options (table_name 'ltable'); +VACUUM ANALYZE; +-- Check statistic interface routine on an empty table. +SELECT * FROM postgres_fdw_stat('ltable'); + postgres_fdw_stat +---------------------------------------------------------------------------------------------------------------------------- + {"namespace" : "public", "relname" : "ltable", "sta_num_slots" : 5, "relpages" : 0, "reltuples" : 0.000000, "attrs" : [ ]} +(1 row) + +SELECT * FROM postgres_fdw_stat('ftable'); +ERROR: Can be used for ordinary relation only. Reltype: f. +-- Check statistic interface routine on non-empty tables. INSERT INTO ltable (a, b) (SELECT *, 1.01 FROM generate_series(1, 1E4)); -VACUUM; --- Check statistic interface routine -SELECT * FROM postgres_fdw_stat('public', 'test', 'a'); - postgres_fdw_stat -------------------- - -(1 row) - -EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) -SELECT * FROM ftable; - QUERY PLAN ---------------------------------------------------------------------------------------------- - Foreign Scan on ftable (cost=100.00..445.00 rows=10000 width=4) (actual rows=10000 loops=1) +ANALYZE ltable, ftable; +SELECT * FROM postgres_fdw_stat('ltable'); + postgres_fdw_stat +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"namespace" : "public", "relname" : "ltable", "sta_num_slots" : 5, "relpages" : 45, "reltuples" : 10000.000000, "attrs" : [{"attname" : "a", "inh" : "false", "nullfrac" : 0.000000, "width" : 4, "distinct" : -1.000000, "stakind" : [2,3,0,0,0], "staop" : [97,97,0,0,0], "stacoll" : [0,0,0,0,0], "nn" : 101, "values1" : [1,100,200,300,400,500,600,700,800,900,1000,1100,1200,1300,1400,1500,1600,1700,1800,1900,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000], "values2" : [ ], "values3" : [ ], "values4" : [ ], "values5" : [ ], "numbers1" : [ ], "nn" : 1, "numbers2" : [1], "numbers3" : [ ], "numbers4" : [ ], "numbers5" : [ ]}, {"attname" : "b", "inh" : "false", "nullfrac" : 0.000000, "width" : 4, "distinct" : 1.000000, "stakind" : [1,3,0,0,0], "staop" : [620,622,0,0,0], "stacoll" : [0,0,0,0,0], "nn" : 1, "values1" : [1.01], "values2" : [ ], "values3" : [ ], "values4" : [ ], "values5" : [ ], "nn" : 1, "numbers1" : [1], "nn" : 1, "numbers2" : [1], "numbers3" : [ ], "numbers4" : [ ], "numbers5" : [ ]}]} +(1 row) + +-- Check ANALYZE on foreign table +INSERT INTO ltable (a, b) (SELECT *, 2.01 FROM generate_series(1E4, 2E4)); +ANALYZE ltable; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable; + QUERY PLAN +---------------------------------------------------------------------------------------- + Seq Scan on ltable (cost=0.00..289.01 rows=20001 width=8) (actual rows=20001 loops=1) +(1 row) + +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable; + QUERY PLAN +---------------------------------------------------------------------------------------------- + Foreign Scan on ftable (cost=100.00..445.00 rows=10000 width=4) (actual rows=20001 loops=1) +(1 row) + +ANALYZE ftable; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable; + QUERY PLAN +---------------------------------------------------------------------------------------------- + Foreign Scan on ftable (cost=100.00..789.03 rows=20001 width=4) (actual rows=20001 loops=1) +(1 row) + +-- Check selectivity +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 10; + QUERY PLAN +---------------------------------------------------------------------------------------- + Seq Scan on ltable (cost=0.00..339.01 rows=19991 width=8) (actual rows=19991 loops=1) + Filter: (a > 10) + Rows Removed by Filter: 10 +(3 rows) + +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 10; + QUERY PLAN +---------------------------------------------------------------------------------------------- + Foreign Scan on ftable (cost=100.00..838.83 rows=19991 width=4) (actual rows=19991 loops=1) +(1 row) + +-- Check new attribute +ALTER TABLE ltable ADD COLUMN c int DEFAULT 42; +ALTER TABLE ftable ADD COLUMN c int; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 10 AND c < 15; + QUERY PLAN +------------------------------------------------------------------------------------ + Seq Scan on ltable (cost=0.00..389.01 rows=6664 width=12) (actual rows=0 loops=1) + Filter: ((a > 10) AND (c < 15)) + Rows Removed by Filter: 20001 +(3 rows) + +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 10 AND c < 15; + QUERY PLAN +----------------------------------------------------------------------------------------- + Foreign Scan on ftable (cost=100.00..622.29 rows=6664 width=8) (actual rows=0 loops=1) +(1 row) + +ANALYZE ltable, ftable; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 10 AND c < 15; + QUERY PLAN +--------------------------------------------------------------------------------- + Seq Scan on ltable (cost=0.00..389.01 rows=1 width=12) (actual rows=0 loops=1) + Filter: ((a > 10) AND (c < 15)) + Rows Removed by Filter: 20001 +(3 rows) + +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 10 AND c < 15; + QUERY PLAN +-------------------------------------------------------------------------------------- + Foreign Scan on ftable (cost=100.00..489.03 rows=1 width=8) (actual rows=0 loops=1) +(1 row) + +-- Test default vacuum analyzes foreign relation +INSERT INTO ltable (a, b) (SELECT *, 2.01 FROM generate_series(2E4, 3E4)); +VACUUM ANALYZE ltable, ftable; -- Don't depend on default order of analysis here. +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 100 AND c < 43; + QUERY PLAN +----------------------------------------------------------------------------------------- + Seq Scan on ltable (cost=0.00..593.03 rows=29902 width=12) (actual rows=29902 loops=1) + Filter: ((a > 100) AND (c < 43)) + Rows Removed by Filter: 100 +(3 rows) + +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 100 AND c < 43; + QUERY PLAN +----------------------------------------------------------------------------------------------- + Foreign Scan on ftable (cost=100.00..1291.07 rows=29902 width=8) (actual rows=29902 loops=1) (1 row) diff --git a/contrib/postgres_fdw/postgres_fdw--1.0--1.1.sql b/contrib/postgres_fdw/postgres_fdw--1.0--1.1.sql index 40d36c95df..ff0dd6f83e 100644 --- a/contrib/postgres_fdw/postgres_fdw--1.0--1.1.sql +++ b/contrib/postgres_fdw/postgres_fdw--1.0--1.1.sql @@ -4,7 +4,7 @@ \echo Use "CREATE EXTENSION postgres_fdw" to load this file. \quit -- Returns serialized stat tuples for the relation attributes -CREATE FUNCTION postgres_fdw_stat(TEXT, TEXT, VARIADIC text[]) +CREATE FUNCTION postgres_fdw_stat(TEXT) RETURNS json AS 'MODULE_PATHNAME' LANGUAGE C STRICT; diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index e4c7bcec49..f81be84c02 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -392,6 +392,7 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *output_rel, void *extra); +static char *postgresGetForeignRelStat(Relation rel); /* * Helper functions @@ -559,6 +560,7 @@ postgres_fdw_handler(PG_FUNCTION_ARGS) /* Support functions for upper relation push-down */ routine->GetForeignUpperPaths = postgresGetForeignUpperPaths; + routine->GetForeignRelStat = postgresGetForeignRelStat; PG_RETURN_POINTER(routine); } @@ -569,13 +571,12 @@ postgres_fdw_handler(PG_FUNCTION_ARGS) Datum postgres_fdw_stat(PG_FUNCTION_ARGS) { - const char *nspname = text_to_cstring(PG_GETARG_TEXT_PP(0)); - const char *relname = text_to_cstring(PG_GETARG_TEXT_PP(1)); - StringInfo result; + const char *relname = text_to_cstring(PG_GETARG_TEXT_PP(0)); + const char *json_stat; - result = makeStringInfo(); + json_stat = relstat_to_json(relname); - PG_RETURN_TEXT_P(cstring_to_text_with_len(result->data, result->len)); + PG_RETURN_TEXT_P(cstring_to_text(json_stat)); } /* @@ -6601,3 +6602,45 @@ find_em_expr_for_input_target(PlannerInfo *root, elog(ERROR, "could not find pathkey item to sort"); return NULL; /* keep compiler quiet */ } + +static char * +postgresGetForeignRelStat(Relation rel) +{ + ForeignTable *table; + UserMapping *user; + PGconn *conn; + unsigned int cursor_number; + StringInfoData sql; + PGresult *volatile res = NULL; + char fetch_sql[64]; + char *json; + + table = GetForeignTable(RelationGetRelid(rel)); + user = GetUserMapping(rel->rd_rel->relowner, table->serverid); + conn = GetConnection(user, false); + + cursor_number = GetCursorNumber(conn); + initStringInfo(&sql); + appendStringInfo(&sql, "DECLARE c%u CURSOR FOR ", cursor_number); + deparseGetStatSql(&sql, rel); + + res = pgfdw_exec_query(conn, sql.data); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + pgfdw_report_error(ERROR, res, conn, false, sql.data); + PQclear(res); + + snprintf(fetch_sql, sizeof(fetch_sql), "FETCH FROM c%u", cursor_number); + res = pgfdw_exec_query(conn, fetch_sql); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + pgfdw_report_error(ERROR, res, conn, false, sql.data); + + Assert(PQntuples(res) == 1); + Assert(PQnfields(res) == 1); + + json = pstrdup(PQgetvalue(res, 0, 0)); + PQclear(res); + close_cursor(conn, cursor_number); + ReleaseConnection(conn); + + return json; +} diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h index eef410db39..bb590bfc21 100644 --- a/contrib/postgres_fdw/postgres_fdw.h +++ b/contrib/postgres_fdw/postgres_fdw.h @@ -190,6 +190,7 @@ extern void deparseDirectDeleteSql(StringInfo buf, PlannerInfo *root, extern void deparseAnalyzeSizeSql(StringInfo buf, Relation rel); extern void deparseAnalyzeSql(StringInfo buf, Relation rel, List **retrieved_attrs); +extern void deparseGetStatSql(StringInfo buf, Relation rel); extern void deparseStringLiteral(StringInfo buf, const char *val); extern Expr *find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel); extern Expr *find_em_expr_for_input_target(PlannerInfo *root, diff --git a/contrib/postgres_fdw/sql/foreign_stat.sql b/contrib/postgres_fdw/sql/foreign_stat.sql index 18f649b477..0829ebc522 100644 --- a/contrib/postgres_fdw/sql/foreign_stat.sql +++ b/contrib/postgres_fdw/sql/foreign_stat.sql @@ -1,10 +1,41 @@ CREATE TABLE ltable (a int, b real); CREATE FOREIGN TABLE ftable (a int) server loopback options (table_name 'ltable'); +VACUUM ANALYZE; + +-- Check statistic interface routine on an empty table. +SELECT * FROM postgres_fdw_stat('ltable'); +SELECT * FROM postgres_fdw_stat('ftable'); + +-- Check statistic interface routine on non-empty tables. INSERT INTO ltable (a, b) (SELECT *, 1.01 FROM generate_series(1, 1E4)); -VACUUM; +ANALYZE ltable, ftable; +SELECT * FROM postgres_fdw_stat('ltable'); + +-- Check ANALYZE on foreign table +INSERT INTO ltable (a, b) (SELECT *, 2.01 FROM generate_series(1E4, 2E4)); +ANALYZE ltable; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable; +ANALYZE ftable; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable; + +-- Check selectivity +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 10; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 10; + +-- Check new attribute +ALTER TABLE ltable ADD COLUMN c int DEFAULT 42; +ALTER TABLE ftable ADD COLUMN c int; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 10 AND c < 15; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 10 AND c < 15; + +ANALYZE ltable, ftable; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 10 AND c < 15; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 10 AND c < 15; --- Check statistic interface routine -SELECT * FROM postgres_fdw_stat('public', 'test', 'a'); +-- Test default vacuum analyzes foreign relation +INSERT INTO ltable (a, b) (SELECT *, 2.01 FROM generate_series(2E4, 3E4)); +VACUUM ANALYZE ltable, ftable; -- Don't depend on default order of analysis here. +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ltable WHERE a > 100 AND c < 43; +EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) SELECT * FROM ftable WHERE a > 100 AND c < 43; -EXPLAIN (TIMING OFF, SUMMARY OFF, COSTS ON, ANALYZE) -SELECT * FROM ftable; diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 8af12b5c6b..ca2642360e 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -30,6 +30,7 @@ #include "catalog/catalog.h" #include "catalog/index.h" #include "catalog/indexing.h" +#include "catalog/namespace.h" #include "catalog/pg_collation.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" @@ -38,8 +39,10 @@ #include "commands/progress.h" #include "commands/tablecmds.h" #include "commands/vacuum.h" +#include "common/jsonapi.h" #include "executor/executor.h" #include "foreign/fdwapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "parser/parse_oper.h" @@ -58,9 +61,11 @@ #include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/guc.h" +#include "utils/json.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" +#include "utils/regproc.h" #include "utils/sampling.h" #include "utils/sortsupport.h" #include "utils/syscache.h" @@ -107,6 +112,10 @@ static void update_attstats(Oid relid, bool inh, static Datum std_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull); static Datum ind_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull); +static void parse_statjson(Relation rel, char *buffer, size_t size, + VacAttrStats **vacattrstats, int *natts, + BlockNumber *relpages, double *reltuples); +static bool update_foreign_relation_stat(Relation rel); /* * analyze_rel() -- analyze one relation @@ -193,6 +202,17 @@ analyze_rel(Oid relid, RangeVar *relation, return; } + if (onerel->rd_rel->relkind == RELKIND_FOREIGN_TABLE && + update_foreign_relation_stat(onerel)) + { + /* + * The foreign relation statistics was updated by existed stat tuple + * from remote server. + */ + relation_close(onerel, ShareUpdateExclusiveLock); + return; + } + /* * Check that it's of an analyzable relkind, and set up appropriately. */ @@ -1421,6 +1441,566 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, return numrows; } +typedef enum +{ + JS_EXPECT_NSPNAME_VALUE, + JS_EXPECT_RELNAME_VALUE, + JS_EXPECT_SLOTSNUM_VALUE, + JS_EXPECT_RELPAGES_VALUE, + JS_EXPECT_RELTUPLES_VALUE, + JS_EXPECT_ATTNAME_FIELD, + JS_EXPECT_ATTNAME_VALUE, + JS_EXPECT_INH_VALUE, + JS_EXPECT_NULLFRAC_VALUE, + JS_EXPECT_WIDTH_VALUE, + JS_EXPECT_DISTINCT_VALUE, + JS_EXPECT_KIND_VALUE, + JS_EXPECT_STAOP_VALUE, + JS_EXPECT_STACOLL_VALUE, + JS_EXPECT_ARRSIZE_VALUE, + JS_EXPECT_NUMBERS_VALUE, + JS_EXPECT_VALUES_VALUE, + JS_EXPECT_PARAM_FIELD +} JsonStatSemanticState; + +typedef struct JsonStatParseState +{ + Relation rel; + int natts; + int state; + int arraysize; /* size of the array */ + int arrayval; /* current position in the array */ + int numbersN; + int valuesN; + + char *nspname; + char *relname; + + BlockNumber relpages; + double reltuples; + VacAttrStats **vas; +} JsonStatParseState; + +static bool skip_attr = false; + +static void +json_stat_object_field_start(void *state, char *fname, bool isnull) +{ + JsonStatParseState *parse = (JsonStatParseState *) state; + int attnum = parse->natts - 1; + + if (skip_attr && strcmp(fname, "attname") != 0) + return; + + Assert(parse->state == JS_EXPECT_PARAM_FIELD || + parse->state == JS_EXPECT_ATTNAME_FIELD); + Assert(parse->arrayval < 0); + + if (strcmp(fname, "namespace") == 0) + parse->state = JS_EXPECT_NSPNAME_VALUE; + else if (strcmp(fname, "relname") == 0) + parse->state = JS_EXPECT_RELNAME_VALUE; + else if (strcmp(fname, "sta_num_slots") == 0) + parse->state = JS_EXPECT_SLOTSNUM_VALUE; + else if (strcmp(fname, "relpages") == 0) + parse->state = JS_EXPECT_RELPAGES_VALUE; + else if (strcmp(fname, "reltuples") == 0) + parse->state = JS_EXPECT_RELTUPLES_VALUE; + else if(strcmp(fname, "attrs") == 0) + parse->state = JS_EXPECT_ATTNAME_FIELD; + else if (strcmp(fname, "attname") == 0) + { + parse->state = JS_EXPECT_ATTNAME_VALUE; + skip_attr = false; + } + else if (strcmp(fname, "nullfrac") == 0) + parse->state = JS_EXPECT_NULLFRAC_VALUE; + else if (strcmp(fname, "inh") == 0) + parse->state = JS_EXPECT_INH_VALUE; + else if (strcmp(fname, "width") == 0) + parse->state = JS_EXPECT_WIDTH_VALUE; + else if (strcmp(fname, "distinct") == 0) + parse->state = JS_EXPECT_DISTINCT_VALUE; + else if (strcmp(fname, "nn") == 0) + { + Assert(parse->arraysize == 0); + parse->state = JS_EXPECT_ARRSIZE_VALUE; + } + else + { + parse->arrayval = 0; + if (strcmp(fname, "stakind") == 0) + parse->state = JS_EXPECT_KIND_VALUE; + else if (strcmp(fname, "staop") == 0) + parse->state = JS_EXPECT_STAOP_VALUE; + else if (strcmp(fname, "stacoll") == 0) + parse->state = JS_EXPECT_STACOLL_VALUE; + else if (strstr(fname, "numbers") != NULL) + { + parse->numbersN = atoi(&fname[7]); + Assert(parse->numbersN > 0 && parse->numbersN <= 5); + if (parse->arraysize > 0) + parse->vas[attnum]->stanumbers[parse->numbersN-1] = + (float4 *) palloc(parse->arraysize * sizeof(float4)); + else + parse->vas[attnum]->stanumbers[parse->numbersN-1] = NULL; + + parse->vas[attnum]->numnumbers[parse->numbersN-1] = parse->arraysize; + parse->state = JS_EXPECT_NUMBERS_VALUE; + } + else if (strstr(fname, "values") != NULL) + { + parse->valuesN = atoi(&fname[6]); + Assert(parse->valuesN > 0 && parse->valuesN <= 5); + + if (parse->arraysize > 0) + parse->vas[parse->natts - 1]->stavalues[parse->valuesN-1] = + (Datum *) palloc(parse->arraysize * sizeof(Datum)); + else + parse->vas[parse->natts - 1]->stavalues[parse->valuesN-1] = NULL; + + parse->vas[attnum]->numvalues[parse->valuesN-1] = parse->arraysize; + parse->state = JS_EXPECT_VALUES_VALUE; + } + else + elog(ERROR, "Unknown stat parameter in JSON string: %s", fname); + } +} + +static void +json_stat_array_end(void *state) +{ + JsonStatParseState *parse = (JsonStatParseState *) state; + parse->arrayval = -1; + parse->arraysize = 0; + parse->numbersN = 0; + parse->valuesN = 0; +} + +static void +json_stat_array_element_end(void *state, bool isnull) +{ + JsonStatParseState *parse = (JsonStatParseState *) state; + + Assert(!isnull); + + if (parse->arrayval < 0) // Debug + return; + + parse->arrayval++; +} + +static void +json_stat_field_end(void *state, char *fname, bool isnull) +{ + JsonStatParseState *parse = (JsonStatParseState *) state; + + Assert(!isnull && parse->arrayval < 0); + parse->state = JS_EXPECT_PARAM_FIELD; +} + +static void +json_stat_scalar(void *state, char *token, JsonTokenType tokentype) +{ + JsonStatParseState *parse = (JsonStatParseState *) state; + Datum value; + int attnum = parse->natts - 1; + + if (skip_attr) + return; + + switch (parse->state) + { + case JS_EXPECT_NSPNAME_VALUE: + Assert(tokentype == JSON_TOKEN_STRING); + parse->nspname = token; + break; + + case JS_EXPECT_RELNAME_VALUE: + Assert(tokentype == JSON_TOKEN_STRING); + parse->relname = token; + break; + + case JS_EXPECT_SLOTSNUM_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER); + if (strcmp(token, "5") != 0) + elog(ERROR, "Incompatible PostgreSQL version"); + break; + + case JS_EXPECT_RELPAGES_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER); + value = DirectFunctionCall1(int4in, CStringGetDatum(token)); + parse->relpages = DatumGetInt32(value); + break; + + case JS_EXPECT_RELTUPLES_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER); + value = DirectFunctionCall1(float4in, CStringGetDatum(token)); + parse->reltuples = DatumGetFloat4(value); + break; + + case JS_EXPECT_ATTNAME_FIELD: + Assert(0); + break; + + case JS_EXPECT_ATTNAME_VALUE: + { + int i; + + Assert(tokentype == JSON_TOKEN_STRING); + + for (i = 0; i < parse->rel->rd_att->natts; i++) + { + if (strcmp(NameStr(parse->rel->rd_att->attrs[i].attname), token) == 0) + break; + } + + if (i == parse->rel->rd_att->natts) + { + /* + * It is a valid case when a foreign table doesn't have all the + * attributes from the base relation. + */ + skip_attr = true; + parse->state = JS_EXPECT_ATTNAME_FIELD; + } + else + { + /* Initialize new storage for the attribute. */ + parse->natts++; + attnum++; + parse->vas[attnum] = examine_attribute(parse->rel, i+1, NULL); + parse->vas[attnum]->stats_valid = true; + Assert(parse->vas[attnum] != NULL); + } + } + break; + + case JS_EXPECT_INH_VALUE: + Assert(tokentype == JSON_TOKEN_STRING); + /* XXX */ + break; + + case JS_EXPECT_WIDTH_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER); + value = DirectFunctionCall1(int4in, CStringGetDatum(token)); + parse->vas[attnum]->stawidth = DatumGetInt32(value); + break; + + case JS_EXPECT_DISTINCT_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER); + value = DirectFunctionCall1(float4in, CStringGetDatum(token)); + parse->vas[attnum]->stadistinct = DatumGetFloat4(value); + break; + + case JS_EXPECT_NULLFRAC_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER); + value = DirectFunctionCall1(float4in, CStringGetDatum(token)); + parse->vas[attnum]->stanullfrac = DatumGetFloat4(value); + break; + + case JS_EXPECT_KIND_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER && parse->arrayval >= 0); + value = DirectFunctionCall1(int2in, CStringGetDatum(token)); + parse->vas[attnum]->stakind[parse->arrayval] = DatumGetInt16(value); + break; + + case JS_EXPECT_STAOP_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER && parse->arrayval >= 0); + value = DirectFunctionCall1(oidin, CStringGetDatum(token)); + parse->vas[attnum]->staop[parse->arrayval] = DatumGetObjectId(value); + break; + + case JS_EXPECT_STACOLL_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER && parse->arrayval >= 0); + value = DirectFunctionCall1(oidin, CStringGetDatum(token)); + parse->vas[attnum]->stacoll[parse->arrayval] = DatumGetObjectId(value); + break; + + case JS_EXPECT_ARRSIZE_VALUE: + Assert(tokentype == JSON_TOKEN_NUMBER && parse->arrayval < 0); + value = DirectFunctionCall1(int4in, CStringGetDatum(token)); + parse->arraysize = DatumGetInt32(value); + Assert(parse->arraysize > 0); + break; + + case JS_EXPECT_NUMBERS_VALUE: + { + int n = parse->numbersN; + int m = parse->arrayval; + + Assert(parse->valuesN == 0); + Assert(tokentype == JSON_TOKEN_NUMBER && n > 0 && n <= 5); + Assert(m >= 0 && m < parse->arraysize); + + value = DirectFunctionCall1(float4in, CStringGetDatum(token)); + parse->vas[attnum]->stanumbers[n-1][m] = DatumGetFloat4(value); + } + break; + + case JS_EXPECT_VALUES_VALUE: + { + int n = parse->valuesN; + int m = parse->arrayval; + Form_pg_attribute att = parse->vas[attnum]->attr; + Oid typinput; + Oid typioparam; + + Assert(parse->numbersN == 0); + Assert(tokentype == JSON_TOKEN_NUMBER && n > 0 && n <= 5); + Assert(m >= 0 && m < parse->arraysize); + + getTypeInputInfo(att->atttypid, &typinput, &typioparam); + parse->vas[attnum]->stavalues[n-1][m] = + OidInputFunctionCall(typinput, token, + typioparam, att->atttypmod); + } + break; + + default: + elog(ERROR, "Unexpected token type: %d. Token: %s. State: %d.", + tokentype, token, parse->state); + } +} + +static void +parse_statjson(Relation rel, char *buffer, size_t size, + VacAttrStats **vacattrstats, int *natts, + BlockNumber *relpages, double *reltuples) +{ + JsonLexContext *lex; + JsonSemAction sem; + JsonStatParseState parse; + + /* Set up our private parsing context. */ + parse.state = JS_EXPECT_PARAM_FIELD; + parse.arraysize = -1; + parse.arrayval = -1; + parse.numbersN = -1; + parse.vas = vacattrstats; + parse.natts = 0; + parse.rel = rel; + skip_attr = false; + + /* Create a JSON lexing context. */ + lex = makeJsonLexContextCstringLen(buffer, size, PG_UTF8, true); + + /* Set up semantic actions. */ + sem.semstate = &parse; + sem.object_start = NULL; + sem.object_end = NULL; + sem.array_start = NULL; + sem.array_end = json_stat_array_end; + sem.object_field_start = json_stat_object_field_start; + sem.object_field_end = json_stat_field_end; + sem.array_element_start = NULL; + sem.array_element_end = json_stat_array_element_end; + sem.scalar = json_stat_scalar; + + /* Run the actual JSON parser. */ + pg_parse_json(lex, &sem); + *natts = parse.natts; + *relpages = parse.relpages; + *reltuples = parse.reltuples; +} + +static bool +update_foreign_relation_stat(Relation rel) +{ + VacAttrStats **vacattrstats = (VacAttrStats **) + palloc0(rel->rd_att->natts * sizeof(VacAttrStats *)); + char *statstr; + int natts = 0; + FdwRoutine *fdwroutine; + BlockNumber relallvisible; + BlockNumber numpages; + double numtuples; + + fdwroutine = GetFdwRoutineForRelation(rel, false); + Assert(fdwroutine != NULL); + + if (fdwroutine->GetForeignRelStat == NULL || strcmp(NameStr(rel->rd_rel->relname), "ftable") != 0) + return false; + + statstr = fdwroutine->GetForeignRelStat(rel); + + if (!statstr) + return false; + /* + * Extract statistics from the JSON string. + */ + parse_statjson(rel, statstr, strlen(statstr), vacattrstats, &natts, &numpages, &numtuples); + + update_attstats(RelationGetRelid(rel), false, natts, vacattrstats); + + visibilitymap_count(rel, &relallvisible, NULL); + vac_update_relstats(rel, numpages, numtuples, relallvisible, false, + InvalidTransactionId, InvalidMultiXactId, false); + + pfree(vacattrstats); + return true; +} + +/* + * relstat_to_json + * + */ +const char * +relstat_to_json(const char *relname) +{ + RangeVar *relvar; + Relation rel; + Relation sd; + List *relname_list; + int attno; + StringInfo str = makeStringInfo(); + int attnum = 0; + + relname_list = stringToQualifiedNameList(relname); + relvar = makeRangeVarFromNameList(relname_list); + rel = relation_openrv(relvar, AccessShareLock); + + if (rel->rd_rel->relkind != RELKIND_RELATION) + { + relation_close(rel, AccessShareLock); + elog(ERROR, + "Can be used for ordinary relation only. Reltype: %c.", + rel->rd_rel->relkind); + } + + /* JSON header of overall relation statistics. */ + appendStringInfoString(str, "{"); + appendStringInfo(str, "\"namespace\" : \"%s\", \"relname\" : \"%s\", \"sta_num_slots\" : %d, ", + get_namespace_name(rel->rd_rel->relnamespace), + NameStr(rel->rd_rel->relname), STATISTIC_NUM_SLOTS); + + appendStringInfo(str, "\"relpages\" : %u, \"reltuples\" : %f, ", + rel->rd_rel->relpages, rel->rd_rel->reltuples); + + appendStringInfo(str, "\"attrs\" : ["); + sd = table_open(StatisticRelationId, RowExclusiveLock); + + for (attno = 0; attno < rel->rd_att->natts; attno++) + { + HeapTuple stup; + Datum values[Natts_pg_statistic]; + bool nulls[Natts_pg_statistic]; + int i, k; + + stup = SearchSysCache3(STATRELATTINH, + ObjectIdGetDatum(RelationGetRelid(rel)), + Int16GetDatum(rel->rd_att->attrs[attno].attnum), + BoolGetDatum(false)); + + if (!HeapTupleIsValid(stup)) + /* Go to the next attribute, if we haven't statistics for. */ + continue; + + if (attnum++ > 0) + appendStringInfoString(str, ", "); + + heap_deform_tuple(stup, RelationGetDescr(sd), values, nulls); + + /* JSON header of attrribute statistics. */ + appendStringInfo(str, "{\"attname\" : \"%s\", \"inh\" : \"%s\", \"nullfrac\" : %f, \"width\" : %d, \"distinct\" : %f, ", + NameStr(*attnumAttName(rel, rel->rd_att->attrs[attno].attnum)), + DatumGetBool(values[Anum_pg_statistic_stainherit - 1]) ? "true" : "false", + DatumGetFloat4(values[Anum_pg_statistic_stanullfrac - 1]), + DatumGetInt32(values[Anum_pg_statistic_stawidth - 1]), + DatumGetFloat4(values[Anum_pg_statistic_stadistinct - 1])); + + appendStringInfoString(str, "\"stakind\" : ["); + i = Anum_pg_statistic_stakind1 - 1; + for (k = 0; k < STATISTIC_NUM_SLOTS; k++) + { + appendStringInfo(str, "%d", DatumGetInt16(values[i++])); + if (k < STATISTIC_NUM_SLOTS - 1) + appendStringInfo(str, ","); + } + appendStringInfoString(str, "], \"staop\" : ["); + + i = Anum_pg_statistic_staop1 - 1; + for (k = 0; k < STATISTIC_NUM_SLOTS; k++) + { + appendStringInfo(str, "%u", DatumGetObjectId(values[i++])); + if (k < STATISTIC_NUM_SLOTS - 1) + appendStringInfoChar(str, ','); + } + appendStringInfoString(str, "], \"stacoll\" : ["); + + i = Anum_pg_statistic_stacoll1 - 1; + for (k = 0; k < STATISTIC_NUM_SLOTS; k++) + { + appendStringInfo(str, "%u", DatumGetObjectId(values[i++])); + if (k < STATISTIC_NUM_SLOTS - 1) + appendStringInfoChar(str, ','); + } + + appendStringInfoString(str, "], "); + + for (k = 0; k < STATISTIC_NUM_SLOTS; k++) + { + bool isnull; + Datum val; + + val = SysCacheGetAttr(STATRELATTINH, stup, + Anum_pg_statistic_stavalues1 + k, + &isnull); + + if (isnull) + appendStringInfo(str, "\"values%d\" : [ ]", k+1); + else + { + ArrayType *v = DatumGetArrayTypeP(val); + + appendStringInfo(str, "\"nn\" : %d, ", + ArrayGetNItems(ARR_NDIM(v), ARR_DIMS(v))); + appendStringInfo(str, "\"values%d\" : ", k+1); + arr_to_json(val, str); + } + appendStringInfoString(str, ", "); + } + + /* --- Extract numbers --- */ + for (k = 0; k < STATISTIC_NUM_SLOTS; k++) + { + bool isnull; + Datum val; + + if (k > 0) + appendStringInfoString(str, ", "); + + val = SysCacheGetAttr(STATRELATTINH, stup, + Anum_pg_statistic_stanumbers1 + k, + &isnull); + if (isnull) + appendStringInfo(str, "\"numbers%d\" : [ ]", k+1); + else + { + ArrayType *v = DatumGetArrayTypeP(val); + + appendStringInfo(str, "\"nn\" : %d, ", + ArrayGetNItems(ARR_NDIM(v), ARR_DIMS(v))); + appendStringInfo(str, "\"numbers%d\" : ", k+1); + arr_to_json(val, str); + } + } + + appendStringInfoString(str, "}"); + ReleaseSysCache(stup); + } + + if (attnum == 0) + appendStringInfoString(str, " ]"); + else + appendStringInfoChar(str, ']'); + appendStringInfoString(str, "}"); + + table_close(sd, RowExclusiveLock); + relation_close(rel, AccessShareLock); + + return str->data; +} /* * update_attstats() -- update attribute statistics for one relation diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 23eb605d4c..963ee51523 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -882,6 +882,7 @@ get_all_vacuum_rels(int options) * them. */ if (classForm->relkind != RELKIND_RELATION && + classForm->relkind != RELKIND_FOREIGN_TABLE && classForm->relkind != RELKIND_MATVIEW && classForm->relkind != RELKIND_PARTITIONED_TABLE) continue; @@ -1795,7 +1796,17 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params) /* * Check that it's of a vacuumable relkind. */ - if (onerel->rd_rel->relkind != RELKIND_RELATION && + if (onerel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + relation_close(onerel, lmode); + PopActiveSnapshot(); + CommitTransactionCommand(); + if (params->options & VACOPT_ANALYZE) + return true; + else + return false; + } + else if (onerel->rd_rel->relkind != RELKIND_RELATION && onerel->rd_rel->relkind != RELKIND_MATVIEW && onerel->rd_rel->relkind != RELKIND_TOASTVALUE && onerel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) diff --git a/src/backend/utils/adt/json.c b/src/backend/utils/adt/json.c index a7a91b72f6..adea67a18d 100644 --- a/src/backend/utils/adt/json.c +++ b/src/backend/utils/adt/json.c @@ -656,6 +656,12 @@ add_json(Datum val, bool is_null, StringInfo result, datum_to_json(val, is_null, result, tcategory, outfuncoid, key_scalar); } +void +arr_to_json(Datum array, StringInfo result) +{ + array_to_json_internal(array, result, false); +} + /* * SQL function array_to_json(row) */ diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index a4cd721400..9c1b89d5b6 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -283,6 +283,7 @@ extern Relation vacuum_open_relation(Oid relid, RangeVar *relation, extern void analyze_rel(Oid relid, RangeVar *relation, VacuumParams *params, List *va_cols, bool in_outer_xact, BufferAccessStrategy bstrategy); +extern const char *relstat_to_json(const char *relname); extern bool std_typanalyze(VacAttrStats *stats); /* in utils/misc/sampling.c --- duplicate of declarations in utils/sampling.h */ diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index 95556dfb15..51979ab3bf 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -169,6 +169,7 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root, typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root, List *fdw_private, RelOptInfo *child_rel); +typedef char *(*GetForeignRelStat_function) (Relation rel); /* * FdwRoutine is the struct returned by a foreign-data wrapper's handler @@ -246,6 +247,7 @@ typedef struct FdwRoutine /* Support functions for path reparameterization. */ ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild; + GetForeignRelStat_function GetForeignRelStat; } FdwRoutine; diff --git a/src/include/utils/json.h b/src/include/utils/json.h index 4345fbdc31..a6b51bf191 100644 --- a/src/include/utils/json.h +++ b/src/include/utils/json.h @@ -20,5 +20,6 @@ extern void escape_json(StringInfo buf, const char *str); extern char *JsonEncodeDateTime(char *buf, Datum value, Oid typid, const int *tzp); +extern void arr_to_json(Datum array, StringInfo result); #endif /* JSON_H */ -- 2.17.1