pglogical_output - a general purpose logical decoding output plugin
Hi all
I'd like to submit pglogical_output for inclusion in the 9.6 series as
a contrib.
The output plugin is suitable for a number of uses. It's designed
primarily to supply a data stream to drive a logical replication
client running in another PostgreSQL instance, like the pglogical
client discussed at PGConf.EU 2015. However, the plugin can also be
used to drive other replication solutions, message queues, etc.
This isn't an extension, in that it doesn't actually provide any
extension control file, and you can't CREATE EXTENSION it. It's only
usable via the logical decoding interfaces - at the SQL level, or via
the walsender.
See the README.md and DESIGN.md in the attached patch for details on
the plugin. I will follow up with a summary in a separate mail, along
with a few points I'd value input on or want to focus discussion on.
I anticipate that I'll be following up with a few tweaks, but at this
point the plugin is feature-complete, documented and substantially
ready for inclusion as a contrib.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
Attachments:
0001-Add-contrib-pglogical_output-a-logical-decoding-plug.patchtext/x-patch; charset=US-ASCII; name=0001-Add-contrib-pglogical_output-a-logical-decoding-plug.patchDownload
From b92deb54d10ca0de8b70498bc4898ed0873e36bf Mon Sep 17 00:00:00 2001
From: Craig Ringer <craig@2ndquadrant.com>
Date: Mon, 2 Nov 2015 19:34:21 +0800
Subject: [PATCH] Add contrib/pglogical_output, a logical decoding plugin
---
contrib/Makefile | 1 +
contrib/pglogical_output/.gitignore | 6 +
contrib/pglogical_output/DESIGN.md | 124 +++++
contrib/pglogical_output/Makefile | 51 +++
contrib/pglogical_output/README.md | 364 +++++++++++++++
contrib/pglogical_output/examples/hooks/.gitignore | 1 +
contrib/pglogical_output/examples/hooks/Makefile | 20 +
.../examples/hooks/README.pglogical_output_plhooks | 160 +++++++
.../hooks/pglogical_output_plhooks--1.0.sql | 89 ++++
.../examples/hooks/pglogical_output_plhooks.c | 414 +++++++++++++++++
.../hooks/pglogical_output_plhooks.control | 4 +
contrib/pglogical_output/expected/basic.out | 60 +++
contrib/pglogical_output/expected/hooks.out | 95 ++++
contrib/pglogical_output/expected/init.out | 0
contrib/pglogical_output/expected/params.out | 94 ++++
contrib/pglogical_output/expected/pre-clean.out | 0
contrib/pglogical_output/pglogical_config.c | 498 +++++++++++++++++++++
contrib/pglogical_output/pglogical_config.h | 56 +++
contrib/pglogical_output/pglogical_hooks.c | 234 ++++++++++
contrib/pglogical_output/pglogical_hooks.h | 22 +
contrib/pglogical_output/pglogical_output.c | 463 +++++++++++++++++++
contrib/pglogical_output/pglogical_output.h | 98 ++++
contrib/pglogical_output/pglogical_output/README | 7 +
contrib/pglogical_output/pglogical_output/compat.h | 19 +
contrib/pglogical_output/pglogical_output/hooks.h | 74 +++
contrib/pglogical_output/pglogical_proto.c | 484 ++++++++++++++++++++
contrib/pglogical_output/pglogical_proto.h | 36 ++
contrib/pglogical_output/regression.conf | 2 +
contrib/pglogical_output/sql/basic.sql | 49 ++
contrib/pglogical_output/sql/hooks.sql | 86 ++++
contrib/pglogical_output/sql/params.sql | 77 ++++
contrib/pglogical_output/test/Makefile | 6 +
contrib/pglogical_output/test/README.md | 91 ++++
contrib/pglogical_output/test/base.py | 285 ++++++++++++
contrib/pglogical_output/test/pglogical_proto.py | 240 ++++++++++
.../pglogical_output/test/pglogical_protoreader.py | 112 +++++
contrib/pglogical_output/test/test_basic.py | 89 ++++
contrib/pglogical_output/test/test_binary_mode.py | 172 +++++++
contrib/pglogical_output/test/test_filter.py | 182 ++++++++
contrib/pglogical_output/test/test_parameters.py | 80 ++++
.../test/test_replication_origin.py | 327 ++++++++++++++
contrib/pglogical_output/test/test_tuple_fields.py | 159 +++++++
42 files changed, 5431 insertions(+)
create mode 100644 contrib/pglogical_output/.gitignore
create mode 100644 contrib/pglogical_output/DESIGN.md
create mode 100644 contrib/pglogical_output/Makefile
create mode 100644 contrib/pglogical_output/README.md
create mode 100644 contrib/pglogical_output/examples/hooks/.gitignore
create mode 100644 contrib/pglogical_output/examples/hooks/Makefile
create mode 100644 contrib/pglogical_output/examples/hooks/README.pglogical_output_plhooks
create mode 100644 contrib/pglogical_output/examples/hooks/pglogical_output_plhooks--1.0.sql
create mode 100644 contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.c
create mode 100644 contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.control
create mode 100644 contrib/pglogical_output/expected/basic.out
create mode 100644 contrib/pglogical_output/expected/hooks.out
create mode 100644 contrib/pglogical_output/expected/init.out
create mode 100644 contrib/pglogical_output/expected/params.out
create mode 100644 contrib/pglogical_output/expected/pre-clean.out
create mode 100644 contrib/pglogical_output/pglogical_config.c
create mode 100644 contrib/pglogical_output/pglogical_config.h
create mode 100644 contrib/pglogical_output/pglogical_hooks.c
create mode 100644 contrib/pglogical_output/pglogical_hooks.h
create mode 100644 contrib/pglogical_output/pglogical_output.c
create mode 100644 contrib/pglogical_output/pglogical_output.h
create mode 100644 contrib/pglogical_output/pglogical_output/README
create mode 100644 contrib/pglogical_output/pglogical_output/compat.h
create mode 100644 contrib/pglogical_output/pglogical_output/hooks.h
create mode 100644 contrib/pglogical_output/pglogical_proto.c
create mode 100644 contrib/pglogical_output/pglogical_proto.h
create mode 100644 contrib/pglogical_output/regression.conf
create mode 100644 contrib/pglogical_output/sql/basic.sql
create mode 100644 contrib/pglogical_output/sql/hooks.sql
create mode 100644 contrib/pglogical_output/sql/params.sql
create mode 100644 contrib/pglogical_output/test/Makefile
create mode 100644 contrib/pglogical_output/test/README.md
create mode 100644 contrib/pglogical_output/test/base.py
create mode 100644 contrib/pglogical_output/test/pglogical_proto.py
create mode 100644 contrib/pglogical_output/test/pglogical_protoreader.py
create mode 100644 contrib/pglogical_output/test/test_basic.py
create mode 100644 contrib/pglogical_output/test/test_binary_mode.py
create mode 100644 contrib/pglogical_output/test/test_filter.py
create mode 100644 contrib/pglogical_output/test/test_parameters.py
create mode 100644 contrib/pglogical_output/test/test_replication_origin.py
create mode 100644 contrib/pglogical_output/test/test_tuple_fields.py
diff --git a/contrib/Makefile b/contrib/Makefile
index bd251f6..c1e37b2 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -35,6 +35,7 @@ SUBDIRS = \
pg_stat_statements \
pg_trgm \
pgcrypto \
+ pglogical_output \
pgrowlocks \
pgstattuple \
postgres_fdw \
diff --git a/contrib/pglogical_output/.gitignore b/contrib/pglogical_output/.gitignore
new file mode 100644
index 0000000..2322e13
--- /dev/null
+++ b/contrib/pglogical_output/.gitignore
@@ -0,0 +1,6 @@
+pglogical_output.so
+results/
+regression.diffs
+tmp_install/
+tmp_check/
+log/
diff --git a/contrib/pglogical_output/DESIGN.md b/contrib/pglogical_output/DESIGN.md
new file mode 100644
index 0000000..05fb4d1
--- /dev/null
+++ b/contrib/pglogical_output/DESIGN.md
@@ -0,0 +1,124 @@
+# Design decisions
+
+Explanations of why things are done the way they are.
+
+## Why does pglogical_output exist when there's wal2json etc?
+
+`pglogical_output` does plenty more than convert logical decoding change
+messages to a wire format and send them to the client.
+
+It handles format negotiations, sender-side filtering using pluggable hooks
+(and the associated plugin handling), etc. The protocol its self is also
+important, and incorporates elements like binary datum transfer that can't be
+easily or efficiently achieved with json.
+
+## Custom binary protocol
+
+Why do we have a custom binary protocol inside the walsender / copy both protocol,
+rather than using a json message representation?
+
+Speed and compactness. It's expensive to create json, with lots of allocations.
+It's expensive to decode it too. You can't represent raw binary in json, and must
+encode it, which adds considerable overhead for some data types. Using the
+obvious, easy to decode json representations also makes it difficult to do
+later enhancements planned for the protocol and decoder, like caching row
+metadata.
+
+The protocol implementation is fairly well encapsulated, so in future it should
+be possible to emit json instead for clients that request it. Right now that's
+not the priority as tools like wal2json already exist for that.
+
+## Column metadata
+
+The output plugin sends metadata for columsn - at minimum, the column names -
+before each row. It will soon be changed to send the data before each row from
+a new, different table, so that streams of inserts from COPY etc don't repeat
+the metadata each time. That's just a pending feature.
+
+The reason metadata must be sent is that the upstream and downstream table's
+attnos don't necessarily correspond. The column names might, and their ordering
+might even be the same, but any column drop or column type change will result
+in a dropped column on one side. So at the user level the tables look the same,
+but their attnos don't match, and if we rely on attno for replication we'll get
+the wrong data in the wrong columns. Not pretty.
+
+That could be avoided by requiring that the downstream table be strictly
+maintained by DDL replication, but:
+
+* We don't want to require DDL replication
+* That won't work with multiple upstreams feeding into a table
+* The initial table creation still won't be correct if the table has dropped
+ columns, unless we (ab)use `pg_dump`'s `--binary-upgrade` support to emit
+ tables with dropped columns, which we don't want to do.
+
+So despite the bandwidth cost, we need to send metadata.
+
+In future a client-negotiated cache is planned, so that clients can announce
+to the output plugin that they can cache metadata across change series, and
+metadata can only be sent when invalidated by relation changes or when a new
+relation is seen.
+
+Support for type metadata is penciled in to the protocol so that clients that
+don't have table definitions at all - like queueing engines - can decode the
+data. That'll also permit type validation sanity checking on the apply side
+with logical replication.
+
+## Hook entry point as a SQL function
+
+The hooks entry point is a SQL function that populates a passed `internal`
+struct with hook function pointers.
+
+The reason for this is that hooks are specified by a remote peer over the
+network. We can't just let the peer say "dlsym() this arbitrary function name
+and call it with these arguments" for fairly obvious security reasons. At bare
+minimum all replication using hooks would have to be superuser-only if we did
+that.
+
+The SQL entry point is only called once per decoding session and the rest of
+the calls are plain C function pointers.
+
+## The startup reply message
+
+The protocol design choices available to `pg_logical` are constrained by being
+contained in the copy-both protocol within the fe/be protocol, running as a
+logical decoding plugin. The plugin has no direct access to the network socket
+and can't send or receive messages whenever it wants, only under the control of
+the walsender and logical decoding framework.
+
+The only opportunity for the client to send data directly to the logical
+decoding plugin is in the `START_REPLICATION` parameters, and it can't send
+anything to the client before that point.
+
+This means there's no opportunity for a multi-way step negotiation between
+client and server. We have to do all the negotiation we're going to in a single
+exchange of messages - the setup parameters and then the replication start
+message. All the client can do if it doesn't like the offer the server makes is
+disconnect and try again with different parameters.
+
+That's what the startup message is for. It reports the plugin's capabilities
+and tells the client which requested options were honoured. This gives the
+client a chance to decide if it's happy with the output plugin's decision
+or if it wants to reconnect and try again with different options. Iterative
+negotiation, effectively.
+
+## Unrecognised parameters MUST be ignored by client and server
+
+To ensure upward and downward compatibility, the output plugin must ignore
+parameters set by the client if it doesn't recognise them, and the client
+must ignore parameters it doesn't recognise in the server's startup reply
+message.
+
+This ensures that older clients can talk to newer servers and vice versa.
+
+For this to work, the server must never enable new functionality such as
+protocol message types, row formats, etc without the client explicitly
+specifying via a startup parameter that it understands the new functionality.
+Everything must be negotiated.
+
+Similarly, a newer client talking to an older server may ask the server to
+enable functionality, but it can't assume the server will actually honour that
+request. It must check the server's startup reply message to see if the server
+confirmed that it enabled the requested functionality. It might choose to
+disconnect and report an error to the user if the server didn't do what it
+asked. This can be important, e.g. when a security-significant hook is
+specified.
diff --git a/contrib/pglogical_output/Makefile b/contrib/pglogical_output/Makefile
new file mode 100644
index 0000000..8c59eb0
--- /dev/null
+++ b/contrib/pglogical_output/Makefile
@@ -0,0 +1,51 @@
+MODULE_big = pglogical_output
+PGFILEDESC = "pglogical_output - logical replication output plugin"
+
+OBJS = pglogical_output.o pglogical_proto.o pglogical_config.o pglogical_hooks.o
+
+REGRESS = params basic hooks
+
+
+ifdef USE_PGXS
+
+# For regression checks
+# http://www.postgresql.org/message-id/CAB7nPqTsR5o3g-fBi6jbsVdhfPiLFWQ_0cGU5=94Rv_8W3qvFA@mail.gmail.com
+# this makes "make check" give a useful error
+abs_top_builddir = .
+NO_TEMP_INSTALL = yes
+# Usual recipe
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+
+# These don't do anything yet, since temp install is disabled
+EXTRA_INSTALL += ./examples/hooks
+REGRESS_OPTS += --temp-config=regression.conf
+
+plhooks:
+ make -C examples/hooks USE_PGXS=1 clean install
+
+installcheck: plhooks
+
+else
+
+subdir = contrib/pglogical_output
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# 'make installcheck' disabled when building in-tree because these tests
+# require "wal_level=logical", which typical installcheck users do not have
+# (e.g. buildfarm clients).
+installcheck:
+ ;
+
+EXTRA_INSTALL += $(subdir)/examples/hooks
+EXTRA_REGRESS_OPTS += --temp-config=./regression.conf
+
+endif
+
+install: all
+ $(MKDIR_P) '$(DESTDIR)$(includedir)'/pglogical_output
+ $(INSTALL_DATA) pglogical_output/compat.h '$(DESTDIR)$(includedir)'/pglogical_output
+ $(INSTALL_DATA) pglogical_output/hooks.h '$(DESTDIR)$(includedir)'/pglogical_output
diff --git a/contrib/pglogical_output/README.md b/contrib/pglogical_output/README.md
new file mode 100644
index 0000000..d547774
--- /dev/null
+++ b/contrib/pglogical_output/README.md
@@ -0,0 +1,364 @@
+# `pglogical` Output Plugin
+
+This is the [logical decoding](http://www.postgresql.org/docs/current/static/logicaldecoding.html)
+[output plugin](http://www.postgresql.org/docs/current/static/logicaldecoding-output-plugin.html)
+for `pglogical`. Its purpose is to extract a change stream from a PostgreSQL
+database and send it to a client over a network connection using a
+well-defined, efficient protocol that multiple different applications can
+consume.
+
+The primary purpose of `pglogical_output` is to supply data to logical
+streaming replication solutions, but any application can potentially use its
+data stream. The output stream is designed to be compact and fast to decode,
+and the plugin supports upstream filtering of data so that only the required
+information is sent.
+
+No triggers are required to collect the change stream and no external ticker or
+other daemon is required. It's accumulated using
+[replication slots](http://www.postgresql.org/docs/current/static/logicaldecoding-explanation.html#AEN66446),
+as supported in PostgreSQL 9.4 or newer, and sent on top of the
+[PostgreSQL streaming replication protocol](http://www.postgresql.org/docs/current/static/protocol-replication.html).
+
+Unlike block-level ("physical") streaming replication, the change stream from
+the `pglogical` output plugin is compatible across different PostgreSQL
+versions and can even be consumed by non-PostgreSQL clients.
+
+The use of a replication slot means that the change stream is reliable and
+crash-safe. If the client disconnects or crashes it can reconnect and resume
+replay from the last message that client processed. Server-side changes that
+occur while the client is disconnected are accumulated in the queue to be sent
+when the client reconnects. This reliabiliy also means that server-side
+resources are consumed whether or not a client is connected.
+
+# Why another output plugin?
+
+See [`DESIGN.md`](DESIGN.md) for a discussion of why using one of the existing
+generic logical decoding output plugins like `wal2json` to drive a logical
+replication downstream isn't ideal. It's mostly about speed.
+
+# Architecture and high level interaction
+
+The output plugin is loaded by a PostgreSQL walsender process when a client
+connects to PostgreSQL using the PostgreSQL wire protocol with connection
+option `replication=database`, then uses
+[the `CREATE_REPLICATION_SLOT ... LOGICAL ...` or `START_REPLICATION SLOT ... LOGICAL ...` commands](http://www.postgresql.org/docs/current/static/logicaldecoding-walsender.html) to start streaming changes. (It can also be used via
+[SQL level functions](http://www.postgresql.org/docs/current/static/logicaldecoding-sql.html)
+over a non-replication connection, but this is mainly for debugging purposes).
+
+The client supplies parameters to the `START_REPLICATION SLOT ... LOGICAL ...`
+command to specify the version of the `pglogical` protocol it supports,
+whether it wants binary format, etc.
+
+The output plugin processes the connection parameters and the connection enters
+streaming replication protocol mode, sometimes called "COPY BOTH" mode because
+it's based on the protocol used for the `COPY` command. PostgreSQL then calls
+functions in this plugin to send it a stream of transactions to decode and
+translate into network messages. This stream of changes continues until the
+client disconnects.
+
+The only client-to-server interaction after startup is the sending of periodic
+feedback messages that allow the replication slot to discard no-longer-needed
+change history. The client *must* send feedback, otherwise `pg_xlog` on the
+server will eventually fill up and the server will stop working.
+
+
+# Usage
+
+The overall flow of client/server interaction is:
+
+* Client makes PostgreSQL fe/be protocol connection to server
+ * Connection options must include `replication=database` and `dbname=[...]` parameters
+ * The PostgreSQL client library can be `libpq` or anything else that supports the replication sub-protocol
+ * The same mechanisms are used for authentication and protocol encryption as for a normal non-replication connection
+* [Client issues `IDENTIFY_SYSTEM`
+ * Server responds with a single row containing system identity info
+* Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'` if it's setting up for the first time
+ * Server responds with success info and a snapshot identifier
+ * Client may at this point use the snapshot identifier on other connections while leaving this one idle
+* Client issues `START_REPLICATION SLOT slotname LOGICAL 0/0 (...options...)` to start streaming, which loops:
+ * Server emits `pglogical` message block encapsulated in a replication protocol `CopyData` message
+ * Client receives and unwraps message, then decodes the `pglogical` message block
+ * Client intermittently sends a standby status update message to server to confirm replay
+* ... until client sends a graceful connection termination message on the fe/be protocol level or the connection is broken
+
+ The details of `IDENTIFY_SYSTEM`, `CREATE_REPLICATION_SLOT` and `START_REPLICATION` are discussed in the [replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html) and will not be repeated here.
+
+## Make a replication connection
+
+To use the `pglogical` plugin you must first establish a PostgreSQL FE/BE
+protocol connection using the client library of your choice, passing
+`replication=database` as one of the connection parameters. `database` is a
+literal string and is not replaced with the database name; instead the database
+name is passed separately in the usual `dbname` parameter. Note that
+`replication` is not a GUC (configuration parameter) and may not be passed in
+the `options` parameter on the connection, it's a top-level parameter like
+`user` or `dbname`.
+
+Example connection string for `libpq`:
+
+ 'user=postgres replication=database sslmode=verify-full dbname=mydb'
+
+The plug-in name to pass on logical slot creation is `'pglogical'`.
+
+Details are in the replication protocol docs.
+
+## Get system identity
+
+If required you can use the `IDENTIFY_SYSTEM` command, which reports system
+information:
+
+ systemid | timeline | xlogpos | dbname | dboid
+ ---------------------+----------+-----------+--------+-------
+ 6153224364663410513 | 1 | 0/C429C48 | testd | 16385
+ (1 row)
+
+Details in the replication protocol docs.
+
+## Create the slot if required
+
+If your application creates its own slots on first use and hasn't previously
+connected to this database on this system you'll need to create a replication
+slot. This keeps track of the client's replay state even while it's disconnected.
+
+The slot name may be anything your application wants up to a limit of 63
+characters in length. It's strongly advised that the slot name clearly identify
+the application and the host it runs on.
+
+Pass `pglogical` as the plugin name.
+
+e.g.
+
+ CREATE_REPLICATION_SLOT "reporting_host_42" LOGICAL "pglogical";
+
+`CREATE_REPLICATION_SLOT` returns a snapshot identifier that may be used with
+[`SET TRANSACTION SNAPSHOT`](http://www.postgresql.org/docs/current/static/sql-set-transaction.html)
+to see the database's state as of the moment of the slot's creation. The first
+change streamed from the slot will be the change immediately after this
+snapshot was taken. The snapshot is useful when cloning the initial state of a
+database being replicted. Applications that want to see the change stream
+going forward, but don't care about the initial state, can ignore this. The
+snapshot is only valid as long as the connection that issued the
+`CREATE_REPLICATION_SLOT` remains open and has not run another command.
+
+## Send replication parameters
+
+The client now sends:
+
+ START_REPLICATION SLOT "the_slot_name" LOGICAL (
+ 'Expected_encoding', 'UTF8',
+ 'Max_proto_major_version', '1',
+ 'Min_proto_major_version', '1',
+ ...moreparams...
+ );
+
+to start replication.
+
+The parameters are very important for ensuring that the plugin accepts
+the replication request and streams changes in the expected form. `pglogical`
+parameters are discussed in the separate `pglogical` protocol documentation.
+
+## Process the startup message
+
+`pglogical`'s output plugin will send a `CopyData` message containing its
+startup message as the first protocol message. This message contains a
+set of key/value entries describing the capabilities of the upstream output
+plugin, its version and the Pg version, the tuple format options selected,
+etc.
+
+The downstream client may choose to cleanly close the connection and disconnect
+at this point if it doesn't like the reply. It might then inform the user
+or reconnect with different parameters based on what it learned from the
+first connection's startup message.
+
+## Consume the change stream
+
+`pglogical`'s output plugin now sends a continuous series of `CopyData`
+protocol messages, each of which encapsulates a `pglogical` protocol message
+as documented in the separate protocol docs.
+
+These messages provide information about transaction boundaries, changed
+rows, etc.
+
+The stream continues until the client disconnects, the upstream server is
+restarted, the upstream walsender is terminated by admin action, there's
+a network issue, or the connection is otherwise broken.
+
+The client should send periodic feedback messages to the server to acknowledge
+that it's replayed to a given point and let the server release the resources
+it's holding in case that change stream has to be replayed again. See
+["Hot standby feedback message" in the replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html)
+for details.
+
+## Disconnect gracefully
+
+Disconnection works just like any normal client; you use your client library's
+usual method for closing the connection. No special action is required before
+disconnection, though it's usually a good idea to send a final standby status
+message just before you disconnect.
+
+# Tests
+
+There are two sets of tests bundled with `pglogical_output`: the `pg_regress`
+regression tests and some custom Python tests for the protocol.
+
+The `pg_regress` tests check invalid parameter handling and basic
+functionality. They're intended for use by the buildfarm using an in-tree
+`make check`, but may also be run with an out-of-tree PGXS build against an
+existing PostgreSQL install using `make USE_PGXS=1 clean installcheck`.
+
+The Python tests are more comprehensive, and examine the data sent by
+the extension at the protocol level, validating the protocol structure,
+order and contents. They can run using the SQL-level logical decoding
+interface or, with a psycopg2 containing https://github.com/psycopg/psycopg2/pull/322,
+with the walsender / streaming replication protocol. The Python-based tests
+exercise the internal binary format support, too. See `test/README.md` for
+details.
+
+The tests may fail on installations that are not utf-8 encoded because the
+payloads of the binary protocol output will have text in different encodings,
+which aren't visible to psql as text to be decoded. Avoiding anything except
+7-bit ascii in the tests *should* prevent the problem.
+
+# Hooks
+
+`pglogical_output` exposes a number of extension points where applications can
+modify or override its behaviour.
+
+All hooks are called in their own memory context, which lasts for the duration
+of the logical decoding session. They may switch to longer lived contexts if
+needed, but are then responsible for their own cleanup.
+
+## Hook setup function
+
+The downstream must specify the fully-qualified name of a SQL-callable function
+on the server as the value of the `hooks.setup_function` client parameter.
+The SQL signature of this function is
+
+ CREATE OR REPLACE FUNCTION funcname(hooks internal, memory_context internal)
+ RETURNS void STABLE
+ LANGUAGE c AS 'MODULE_PATHNAME';
+
+Permissions are checked. This function must be callable by the user that the
+output plugin is running as.
+
+The function receives a pointer to a newly allocated structure of hook function
+pointers to populate as its first argument. The function must not free the
+argument.
+
+If the hooks need a private data area to store information across calls, the
+setup function should get the `MemoryContext` pointer from the 2nd argument,
+then `MemoryContextAlloc` a struct for the data in that memory context and
+store the pointer to it in `hooks->hooks_private_data`. This will then be
+accessible on future calls to hook functions. It need not be manually freed, as
+the memory context used for logical decoding will free it when it's freed.
+Don't put anything in it that needs manual cleanup.
+
+Each hook has its own C signature (defined below) and the pointers must be
+directly to the functions. Hooks that the client does not wish to set must be
+left null.
+
+An example is provided in `examples/hooks` and the argument structs are defined
+in `pglogical_output/hooks.h`, which is installed into the PostgreSQL source
+tree when the extension is installed.
+
+## Startup hook
+
+The startup hook is called when logical decoding starts.
+
+This hook can inspect the parameters passed by the client to the output
+plugin as in_params. These parameters *must not* be modified.
+
+It can add new parameters to the set to be returned to the client in the
+startup parameters message, by appending to List out_params, which is
+initially NIL. Each element must be a `DefElem` with the param name
+as the `defname` and a `String` value as the arg, as created with
+`makeDefElem(...)`. It and its contents must be allocated in the
+logical decoding memory context.
+
+For walsender based decoding the startup hook is called only once, and
+cleanup might not be called at the end of the session.
+
+Multiple decoding sessions, and thus multiple startup hook calls, may happen
+in a session if the SQL interface for logical decoding is being used. In
+that case it's guaranteed that the cleanup hook will be called between each
+startup.
+
+When successfully enabled, the output parameter `hooks.startup_hook_enabled` is
+set to true in the startup reply message.
+
+## Transaction filter hook
+
+The transaction filter hook can exclude entire transactions from being decoded
+and replicated based on the node they originated from.
+
+It is passed a `const TxFilterHookArgs *` containing:
+
+* The hook argument supplied by the client, if any
+* The `RepOriginId` that this transaction originated from
+
+and must return boolean, where true retains the transaction for sending to the
+client and false discards it. (Note that this is the reverse sense of the low
+level logical decoding transaction filter hook).
+
+The hook function must *not* free the argument struct or modify its contents.
+
+The transaction filter hook is only called on PostgreSQL 9.5 and above. It
+is ignored on 9.4.
+
+Note that individual changes within a transaction may have different origins to
+the transaction as a whole; see "Origin filtering" for more details. If a
+transaction is filtered out, all changes are filtered out even if their origins
+differ from that of the transaction as a whole.
+
+When successfully enabled, the output parameter
+`hooks.transaction_filter_enabled` is set to true in the startup reply message.
+
+## Row filter hook
+
+The row filter hook is called for each row. It is passed information about the
+table, the transaction origin, and the row origin.
+
+It is passed a `const RowFilterHookArgs*` containing:
+
+* The hook argument supplied by the client, if any
+* The `Relation` the change affects
+* The change type - 'I'nsert, 'U'pdate or 'D'elete
+
+It can return true to retain this row change, sending it to the client, or
+false to discard it.
+
+The function *must not* free the argument struct or modify its contents.
+
+Note that it is more efficient to exclude whole transactions with the
+transaction filter hook rather than filtering out individual rows.
+
+When successfully enabled, the output parameter
+`hooks.row_filter_enabled` is set to true in the startup reply message.
+
+## Shutdown hook
+
+The shutdown hook is called when a decoding session ends. You can't rely on
+this hook being invoked reliably, since a replication-protocol walsender-based
+session might just terminate. It's mostly useful for cleanup to handle repeated
+invocations under the SQL interface to logical decoding.
+
+You don't need a hook to free memory you allocated, unless you explicitly
+switched to a longer lived memory context like TopMemoryContext. Memory allocated
+in the hook context will be automatically when the decoding session shuts down.
+
+## Hook example
+
+... TODO ...
+
+## Writing hooks in procedural languages
+
+You can write hooks in PL/PgSQL, etc, too.
+
+There's a default hook setup callback `pglogical_output_default_hooks` that
+returns a set of hook functions which call PostgreSQL PL functions and return
+the results. They act as C-to-PL wrappers. The PostgreSQL PL functions to call
+for each hook are found by <XXX how? we don't want to use the hook arg, since
+we want it free to use in the hooks themselves. a new param read by startup
+hook?>
+
+... TODO examples ....
diff --git a/contrib/pglogical_output/examples/hooks/.gitignore b/contrib/pglogical_output/examples/hooks/.gitignore
new file mode 100644
index 0000000..140f8cf
--- /dev/null
+++ b/contrib/pglogical_output/examples/hooks/.gitignore
@@ -0,0 +1 @@
+*.so
diff --git a/contrib/pglogical_output/examples/hooks/Makefile b/contrib/pglogical_output/examples/hooks/Makefile
new file mode 100644
index 0000000..501cb38
--- /dev/null
+++ b/contrib/pglogical_output/examples/hooks/Makefile
@@ -0,0 +1,20 @@
+MODULES = pglogical_output_plhooks
+EXTENSION = pglogical_output_plhooks
+DATA = pglogical_output_plhooks--1.0.sql
+DOCS = README.pglogical_output_plhooks
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+PG_CPPFLAGS = -I../..
+include $(PGXS)
+else
+subdir = contrib/pglogical_output/examples/hooks
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# Allow the hook plugin to see the pglogical_output headers
+# Necessary because !PGXS builds don't respect PG_CPPFLAGS
+override CPPFLAGS := $(CPPFLAGS) -I../..
+endif
diff --git a/contrib/pglogical_output/examples/hooks/README.pglogical_output_plhooks b/contrib/pglogical_output/examples/hooks/README.pglogical_output_plhooks
new file mode 100644
index 0000000..4f0ce81
--- /dev/null
+++ b/contrib/pglogical_output/examples/hooks/README.pglogical_output_plhooks
@@ -0,0 +1,160 @@
+pglogical_output_plhooks is an example module for pglogical_output, showing how
+hooks can be implemented.
+
+It provides C wrappers to allow hooks to be written in any supported PL,
+such as PL/PgSQL.
+
+No effort is made to be efficient. To avoid the need to set up cache
+invalidation handling function calls are done via oid each time, with no
+FmgrInfo caching. Also, memory contexts are reset rather freely. If you
+want efficiency, write your hook in C.
+
+(Catalog timetravel is another reason not to write hooks in PLs; see below).
+
+Simple pointless example
+===
+
+To compile and install, just "make USE_PGXS=1 install". Note that pglogical
+must already be installed so that its headers can be found. You might have
+to set the `PATH` so that `pg_config` can be found.
+
+To use it:
+
+ CREATE EXTENSION pglogical_output_plhooks IN SCHEMA public;
+
+in the target database.
+
+Then create at least one hook procedure, of the supported hooks listed below.
+For the sake of this example we'll use some of the toy examples provided in the
+extension:
+
+* startup function: pglo_plhooks_demo_startup
+* row filter: pglo_plhooks_demo_row_filter
+* txn filter: pglo_plhooks_demo_txn_filter
+* shutdown function: pglo_plhooks_demo_shutdown
+
+Now add some arguments to your pglogical_output client's logical decoding setup
+parameters to specify the hook setup function and to tell
+pglogical_output_plhooks about one or more of the hooks you wish it to run. For
+example you might add the following parameters:
+
+ hooks.setup_function, public.pglo_plhooks_setup_fn,
+ pglo_plhooks.startup_hook, pglo_plhooks_demo_startup,
+ pglo_plhooks.row_filter_hook, pglo_plhooks_demo_row_filter,
+ pglo_plhooks.txn_filter_hook, pglo_plhooks_demo_txn_filter,
+ pglo_plhooks.shutdown_hook, pglo_plhooks_demo_shutdown,
+ pglo_plhooks.client_hook_arg, 'whatever-you-want'
+
+to configure the extension to load its hooks, then configure all the demo hooks.
+
+Why the preference for C hooks?
+===
+
+Speed. The row filter hook is called for *every single row* replicated.
+
+If a hook raises an ERROR then replication will probably stop. You won't be
+able to fix it either, because when you change the hook definition the new
+definition won't be visible in the catalogs at the current replay position due
+to catalog time travel. The old definition that raises an error will keep being
+used. You'll need to remove the problem hook from your logical decoding startup
+parameters, which will disable use the hook entirely, until replay proceeds
+past the point you fixed the problem with the hook function.
+
+Similarly, if you try to add use of a newly defined hook on an existing
+replication slot that hasn't replayed past the point you defined the hook yet,
+you'll get an error complaining that the hook function doesn't exist. Even
+though it clearly does when you look at it in psql. The reason is the same: in
+the time traveled catalogs it really doesn't exist. You have to replay past the
+point the hook was created then enable it. In this case the
+pglogical_output_plhooks startup hook will actually see your functions, but
+fail when it tries to call them during decoding since they'll appear to have
+vanished.
+
+If you write your hooks in C you can redefine them rather more easily, since
+the function definition is not subject to catalog timetravel. More importantly,
+it'll probably be a lot faster. The plhooks code has to do a lot of translation
+to pass information to the PL functions and more to get results back; it also
+has to do a lot of memory allocations and a memory context reset after each
+call. That all adds up.
+
+(You could actually write C functions to be called by this extension, but
+that'd be crazy.)
+
+Available hooks
+===
+
+The four hooks provided by pglogical_output are exposed by the module. See the
+pglogical_output documentation for details on what each hook does and when it
+runs.
+
+A function for each hook must have *exactly* the specified parameters and
+return value, or you'll get an error.
+
+None of the functions may return NULL. If they do you'll get an error.
+
+If you specified `pglo_plhooks.client_hook_arg` in the startup parameters it is
+passed as `client_hook_arg` to all hooks. If not specified the empty string is
+passed.
+
+You can find some toy examples in `pglogical_output_plhooks--1.0.sql`.
+
+
+
+Startup hook
+---
+
+Configured with `pglo_plhooks.startup_hook` startup parameter. Runs when
+logical decoding starts.
+
+Signature *must* be:
+
+ CREATE FUNCTION whatever_funcname(startup_params text[], client_hook_arg text)
+ RETURNS text[]
+
+startup_params is an array of the startup params passed to the pglogical output
+plugin, as alternating key/value elements in text representation.
+
+client_hook_arg is also passed.
+
+The return value is an array of alternating key/value elements forming a set
+of parameters you wish to add to the startup reply message sent by pglogical
+on decoding start. It must not be null; return `ARRAY[]::text[]` if you don't
+want to add any params.
+
+Transaction filter
+---
+
+Called only on 9.5+; ignored on 9.4.
+
+The arguments are the replication origin identifier and the client hook param.
+
+The return value is true to keep the transaction, false to discard it.
+
+Signature:
+
+ CREATE FUNCTION whatevername(origin_id int, client_hook_arg text)
+ RETURNS boolean
+
+Row filter
+--
+
+Called for each row. Return true to replicate the row, false to discard it.
+
+Arguments are the oid of the affected relation, and the change type: 'I'nsert,
+'U'pdate or 'D'elete. There is no way to access the change data - columns changed,
+new values, etc.
+
+Signature:
+
+ CREATE FUNCTION whatevername(affected_rel regclass, change_type "char", client_hook_arg text)
+ RETURNS boolean
+
+Shutdown hook
+--
+
+Pretty uninteresting, but included for completeness.
+
+Signature:
+
+ CREATE FUNCTION whatevername(client_hook_arg text)
+ RETURNS void
diff --git a/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks--1.0.sql b/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks--1.0.sql
new file mode 100644
index 0000000..cdd2af3
--- /dev/null
+++ b/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks--1.0.sql
@@ -0,0 +1,89 @@
+\echo Use "CREATE EXTENSION pglogical_output_plhooks" to load this file. \quit
+
+-- Use @extschema@ or leave search_path unchanged, don't use explicit schema
+
+CREATE FUNCTION pglo_plhooks_setup_fn(internal)
+RETURNS void
+STABLE
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+COMMENT ON FUNCTION pglo_plhooks_setup_fn(internal)
+IS 'Register pglogical output pl hooks. See docs for how to specify functions';
+
+--
+-- Called as the startup hook.
+--
+-- There's no useful way to expose the private data segment, so you
+-- just don't get to use that from pl hooks at this point. The C
+-- wrapper will extract a startup param named pglo_plhooks.client_hook_arg
+-- for you and pass it as client_hook_arg to all callbacks, though.
+--
+-- For implementation convenience, a null client_hook_arg is passed
+-- as the empty string.
+--
+-- Must return the empty array, not NULL, if it has nothing to add.
+--
+CREATE FUNCTION pglo_plhooks_demo_startup(startup_params text[], client_hook_arg text)
+RETURNS text[]
+LANGUAGE plpgsql AS $$
+DECLARE
+ elem text;
+ paramname text;
+ paramvalue text;
+BEGIN
+ FOREACH elem IN ARRAY startup_params
+ LOOP
+ IF elem IS NULL THEN
+ RAISE EXCEPTION 'Startup params may not be null';
+ END IF;
+
+ IF paramname IS NULL THEN
+ paramname := elem;
+ ELSIF paramvalue IS NULL THEN
+ paramvalue := elem;
+ ELSE
+ RAISE NOTICE 'got param: % = %', paramname, paramvalue;
+ paramname := NULL;
+ paramvalue := NULL;
+ END IF;
+ END LOOP;
+
+ RETURN ARRAY['pglo_plhooks_demo_startup_ran', 'true', 'otherparam', '42'];
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_txn_filter(origin_id int, client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- Not much to filter on really...
+ RAISE NOTICE 'Got tx with origin %',origin_id;
+ RETURN true;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_row_filter(affected_rel regclass, change_type "char", client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- This is a totally absurd test, since it checks if the upstream user
+ -- doing replication has rights to make modifications that have already
+ -- been committed and are being decoded for replication. Still, it shows
+ -- how the hook works...
+ IF pg_catalog.has_table_privilege(current_user, affected_rel,
+ CASE change_type WHEN 'I' THEN 'INSERT' WHEN 'U' THEN 'UPDATE' WHEN 'D' THEN 'DELETE' END)
+ THEN
+ RETURN true;
+ ELSE
+ RETURN false;
+ END IF;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_shutdown(client_hook_arg text)
+RETURNS void
+LANGUAGE plpgsql AS $$
+BEGIN
+ RAISE NOTICE 'Decoding shutdown';
+END;
+$$
diff --git a/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.c b/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.c
new file mode 100644
index 0000000..a5144f0
--- /dev/null
+++ b/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.c
@@ -0,0 +1,414 @@
+#include "postgres.h"
+
+#include "pglogical_output/hooks.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_type.h"
+
+#include "nodes/makefuncs.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/reorderbuffer.h"
+
+#include "utils/acl.h"
+#include "utils/array.h"
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+
+#include "fmgr.h"
+#include "miscadmin.h"
+
+PG_MODULE_MAGIC;
+
+PGDLLEXPORT extern Datum pglo_plhooks_setup_fn(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglo_plhooks_setup_fn);
+
+void pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args);
+void pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args);
+bool pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args);
+bool pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args);
+
+typedef struct PLHPrivate
+{
+ const char *client_arg;
+ Oid startup_hook;
+ Oid shutdown_hook;
+ Oid row_filter_hook;
+ Oid txn_filter_hook;
+ MemoryContext hook_call_context;
+} PLHPrivate;
+
+static void read_parameters(PLHPrivate *private, List *in_params);
+static Oid find_startup_hook(const char *proname);
+static Oid find_shutdown_hook(const char *proname);
+static Oid find_row_filter_hook(const char *proname);
+static Oid find_txn_filter_hook(const char *proname);
+static void exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params);
+
+void
+pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args)
+{
+ PLHPrivate *private;
+
+ /* pglogical_output promises to call us in a tx */
+ Assert(IsTransactionState());
+
+ /* Allocated in hook memory context, scoped to the logical decoding session: */
+ startup_args->private_data = private = (PLHPrivate*)palloc(sizeof(PLHPrivate));
+
+ private->startup_hook = InvalidOid;
+ private->shutdown_hook = InvalidOid;
+ private->row_filter_hook = InvalidOid;
+ private->txn_filter_hook = InvalidOid;
+ /* client_arg is the empty string when not specified to simplify function calls */
+ private->client_arg = "";
+
+ read_parameters(private, startup_args->in_params);
+
+ private->hook_call_context = AllocSetContextCreate(CurrentMemoryContext,
+ "pglogical_output plhooks hook call context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+
+ if (private->startup_hook != InvalidOid)
+ exec_user_startup_hook(private, startup_args->in_params, &startup_args->out_params);
+}
+
+void
+pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args)
+{
+ PLHPrivate *private = (PLHPrivate*)shutdown_args->private_data;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->shutdown_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo shutdown hook with %s", private->client_arg);
+ (void) OidFunctionCall1(
+ private->shutdown_hook,
+ CStringGetTextDatum(private->client_arg));
+ elog(DEBUG3, "called pglo shutdown hook");
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+}
+
+bool
+pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)rowfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->row_filter_hook))
+ {
+ char change_type;
+ switch (rowfilter_args->change_type)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ change_type = 'I';
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ change_type = 'U';
+ break;
+ case REORDER_BUFFER_CHANGE_DELETE:
+ change_type = 'D';
+ break;
+ default:
+ elog(ERROR, "unknown change type %d", rowfilter_args->change_type);
+ change_type = '0'; /* silence compiler */
+ }
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo row filter hook with (%u,%c,%s)",
+ rowfilter_args->changed_rel->rd_id, change_type,
+ private->client_arg);
+ ret = DatumGetBool(OidFunctionCall3(
+ private->row_filter_hook,
+ ObjectIdGetDatum(rowfilter_args->changed_rel->rd_id),
+ CharGetDatum(change_type),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "called pglo row filter hook, returns %d", (int)ret);
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+bool
+pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)txnfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+
+ if (OidIsValid(private->txn_filter_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ elog(DEBUG3, "calling pglo txn filter hook with (%hu,%s)",
+ txnfilter_args->origin_id, private->client_arg);
+ ret = DatumGetBool(OidFunctionCall2(
+ private->txn_filter_hook,
+ UInt16GetDatum(txnfilter_args->origin_id),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "calling pglo txn filter hook, returns %d", (int)ret);
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+Datum
+pglo_plhooks_setup_fn(PG_FUNCTION_ARGS)
+{
+ struct PGLogicalHooks *hooks = (struct PGLogicalHooks*) PG_GETARG_POINTER(0);
+
+ /* Your code doesn't need this, it's just for the tests: */
+ Assert(hooks != NULL);
+ Assert(hooks->hooks_private_data == NULL);
+ Assert(hooks->startup_hook == NULL);
+ Assert(hooks->shutdown_hook == NULL);
+ Assert(hooks->row_filter_hook == NULL);
+ Assert(hooks->txn_filter_hook == NULL);
+
+ /*
+ * Just assign the hook pointers. We're not meant to do much
+ * work here.
+ *
+ * Note that private_data is left untouched, to be set up by the
+ * startup hook.
+ */
+ hooks->startup_hook = pglo_plhooks_startup;
+ hooks->shutdown_hook = pglo_plhooks_shutdown;
+ hooks->row_filter_hook = pglo_plhooks_row_filter;
+ hooks->txn_filter_hook = pglo_plhooks_txn_filter;
+ elog(DEBUG3, "configured pglo hooks");
+
+ PG_RETURN_VOID();
+}
+
+static void
+exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params)
+{
+ ArrayType *startup_params;
+ Datum ret;
+ ListCell *lc;
+ Datum *startup_params_elems;
+ bool *startup_params_isnulls;
+ int n_startup_params;
+ int i;
+ MemoryContext old_ctx;
+
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ /*
+ * Build the input parameter array. NULL parameters are passed as the
+ * empty string for the sake of convenience. Each param is two
+ * elements, a key then a value element.
+ */
+ n_startup_params = list_length(in_params) * 2;
+ startup_params_elems = (Datum*)palloc0(sizeof(Datum)*n_startup_params);
+
+ i = 0;
+ foreach (lc, in_params)
+ {
+ DefElem * elem = (DefElem*)lfirst(lc);
+ const char *val;
+
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ val = "";
+ else
+ val = strVal(elem->arg);
+
+ startup_params_elems[i++] = CStringGetTextDatum(elem->defname);
+ startup_params_elems[i++] = CStringGetTextDatum(val);
+ }
+ Assert(i == n_startup_params);
+
+ startup_params = construct_array(startup_params_elems, n_startup_params,
+ TEXTOID, -1, false, 'i');
+
+ ret = OidFunctionCall2(
+ private->startup_hook,
+ PointerGetDatum(startup_params),
+ CStringGetTextDatum(private->client_arg));
+
+ /*
+ * deconstruct return array and add pairs of results to a DefElem list.
+ */
+ deconstruct_array(DatumGetArrayTypeP(ret), TEXTARRAYOID,
+ -1, false, 'i', &startup_params_elems, &startup_params_isnulls,
+ &n_startup_params);
+
+
+ *out_params = NIL;
+ for (i = 0; i < n_startup_params; i = i + 2)
+ {
+ char *value;
+ DefElem *elem;
+
+ if (startup_params_isnulls[i])
+ elog(ERROR, "Array entry corresponding to a key was null at idx=%d", i);
+
+ if (startup_params_isnulls[i+1])
+ value = "";
+ else
+ value = TextDatumGetCString(startup_params_elems[i+1]);
+
+ elem = makeDefElem(
+ TextDatumGetCString(startup_params_elems[i]),
+ (Node*)makeString(value));
+
+ *out_params = lcons(elem, *out_params);
+ }
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+}
+
+static void
+read_parameters(PLHPrivate *private, List *in_params)
+{
+ ListCell *option;
+
+ foreach(option, in_params)
+ {
+ DefElem *elem = lfirst(option);
+
+ if (pg_strcasecmp("pglo_plhooks.client_hook_arg", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.client_hook_arg may not be NULL");
+ private->client_arg = pstrdup(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.startup_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.startup_hook may not be NULL");
+ private->startup_hook = find_startup_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.shutdown_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.shutdown_hook may not be NULL");
+ private->shutdown_hook = find_shutdown_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.txn_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.txn_filter_hook may not be NULL");
+ private->txn_filter_hook = find_txn_filter_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.row_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.row_filter_hook may not be NULL");
+ private->row_filter_hook = find_row_filter_hook(strVal(elem->arg));
+ }
+ }
+}
+
+static Oid
+find_hook_fn(const char *funcname, Oid funcargtypes[], int nfuncargtypes, Oid returntype)
+{
+ Oid funcid;
+ List *qname;
+
+ qname = stringToQualifiedNameList(funcname);
+
+ /* find the the function */
+ funcid = LookupFuncName(qname, nfuncargtypes, funcargtypes, false);
+
+ /* Check expected return type */
+ if (get_func_rettype(funcid) != returntype)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s doesn't return expected type %d",
+ NameListToString(qname), returntype)));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(qname))));
+ }
+
+ list_free_deep(qname);
+
+ return funcid;
+}
+
+static Oid
+find_startup_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = TEXTARRAYOID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, VOIDOID);
+}
+
+static Oid
+find_shutdown_hook(const char *proname)
+{
+ Oid argtypes[1];
+
+ argtypes[0] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 1, VOIDOID);
+}
+
+static Oid
+find_row_filter_hook(const char *proname)
+{
+ Oid argtypes[3];
+
+ argtypes[0] = REGCLASSOID;
+ argtypes[1] = CHAROID;
+ argtypes[2] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 3, BOOLOID);
+}
+
+static Oid
+find_txn_filter_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = INT4OID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, BOOLOID);
+}
diff --git a/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.control b/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.control
new file mode 100644
index 0000000..647b9ef
--- /dev/null
+++ b/contrib/pglogical_output/examples/hooks/pglogical_output_plhooks.control
@@ -0,0 +1,4 @@
+comment = 'pglogical_output pl hooks'
+default_version = '1.0'
+module_pathname = '$libdir/pglogical_output_plhooks'
+relocatable = false
diff --git a/contrib/pglogical_output/expected/basic.out b/contrib/pglogical_output/expected/basic.out
new file mode 100644
index 0000000..2c6a64e
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic.out
@@ -0,0 +1,60 @@
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ count
+-------
+ 17
+(1 row)
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+ count
+-------
+ 17
+(1 row)
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
diff --git a/contrib/pglogical_output/expected/hooks.out b/contrib/pglogical_output/expected/hooks.out
new file mode 100644
index 0000000..69de857
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks.out
@@ -0,0 +1,95 @@
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+ count
+-------
+ 40
+(1 row)
+
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+ count
+-------
+ 53
+(1 row)
+
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+ERROR: function public.nosuchfunction(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+ERROR: function public.wrong_signature_fn(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/init.out b/contrib/pglogical_output/expected/init.out
new file mode 100644
index 0000000..e69de29
diff --git a/contrib/pglogical_output/expected/params.out b/contrib/pglogical_output/expected/params.out
new file mode 100644
index 0000000..5289686
--- /dev/null
+++ b/contrib/pglogical_output/expected/params.out
@@ -0,0 +1,94 @@
+SET synchronous_commit = on;
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+--
+-- Various invalid parameter combos:
+--
+-- Text mode is not supported
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: logical decoding output plugin "pglogical_output" produces binary output, but "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+ERROR: client sent startup parameters in format 2 but we only support format 1
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: only "UTF8" encoding is supported by this server, client requested bork
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Currently we're sensitive to the encoding name's format (TODO)
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: only "UTF8" encoding is supported by this server, client requested UTF-8
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
diff --git a/contrib/pglogical_output/expected/pre-clean.out b/contrib/pglogical_output/expected/pre-clean.out
new file mode 100644
index 0000000..e69de29
diff --git a/contrib/pglogical_output/pglogical_config.c b/contrib/pglogical_output/pglogical_config.c
new file mode 100644
index 0000000..276a9bc
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.c
@@ -0,0 +1,498 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_config.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_config.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output/compat.h"
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+
+#include "catalog/catversion.h"
+#include "catalog/namespace.h"
+
+#include "mb/pg_wchar.h"
+
+#include "utils/builtins.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+typedef enum PGLogicalOutputParamType
+{
+ OUTPUT_PARAM_TYPE_BOOL,
+ OUTPUT_PARAM_TYPE_UINT32,
+ OUTPUT_PARAM_TYPE_STRING,
+ OUTPUT_PARAM_TYPE_QUALIFIED_NAME
+} PGLogicalOutputParamType;
+
+/* param parsing */
+static Datum get_param_value(DefElem *elem, bool null_ok,
+ PGLogicalOutputParamType type);
+
+static Datum get_param(List *options, const char *name, bool missing_ok,
+ bool null_ok, PGLogicalOutputParamType type,
+ bool *found);
+static bool parse_param_bool(DefElem *elem);
+static uint32 parse_param_uint32(DefElem *elem);
+
+static void
+process_parameters_v1(List *options, PGLogicalOutputData *data);
+
+enum {
+ PARAM_UNRECOGNISED,
+ PARAM_MAX_PROTOCOL_VERSION,
+ PARAM_MIN_PROTOCOL_VERSION,
+ PARAM_EXPECTED_ENCODING,
+ PARAM_BINARY_BIGENDIAN,
+ PARAM_BINARY_SIZEOF_DATUM,
+ PARAM_BINARY_SIZEOF_INT,
+ PARAM_BINARY_SIZEOF_LONG,
+ PARAM_BINARY_FLOAT4BYVAL,
+ PARAM_BINARY_FLOAT8BYVAL,
+ PARAM_BINARY_INTEGER_DATETIMES,
+ PARAM_BINARY_WANT_INTERNAL_BASETYPES,
+ PARAM_BINARY_WANT_BINARY_BASETYPES,
+ PARAM_BINARY_BASETYPES_MAJOR_VERSION,
+ PARAM_PG_VERSION,
+ PARAM_FORWARD_CHANGESETS,
+ PARAM_HOOKS_SETUP_FUNCTION,
+} OutputPluginParamKey;
+
+typedef struct {
+ const char * const paramname;
+ int paramkey;
+} OutputPluginParam;
+
+/* Oh, if only C had switch on strings */
+static OutputPluginParam param_lookup[] = {
+ {"max_proto_version", PARAM_MAX_PROTOCOL_VERSION},
+ {"min_proto_version", PARAM_MIN_PROTOCOL_VERSION},
+ {"expected_encoding", PARAM_EXPECTED_ENCODING},
+ {"binary.bigendian", PARAM_BINARY_BIGENDIAN},
+ {"binary.sizeof_datum", PARAM_BINARY_SIZEOF_DATUM},
+ {"binary.sizeof_int", PARAM_BINARY_SIZEOF_INT},
+ {"binary.sizeof_long", PARAM_BINARY_SIZEOF_LONG},
+ {"binary.float4_byval", PARAM_BINARY_FLOAT4BYVAL},
+ {"binary.float8_byval", PARAM_BINARY_FLOAT8BYVAL},
+ {"binary.integer_datetimes", PARAM_BINARY_INTEGER_DATETIMES},
+ {"binary.want_internal_basetypes", PARAM_BINARY_WANT_INTERNAL_BASETYPES},
+ {"binary.want_binary_basetypes", PARAM_BINARY_WANT_BINARY_BASETYPES},
+ {"binary.basetypes_major_version", PARAM_BINARY_BASETYPES_MAJOR_VERSION},
+ {"pg_version", PARAM_PG_VERSION},
+ {"forward_changesets", PARAM_FORWARD_CHANGESETS},
+ {"hooks.setup_function", PARAM_HOOKS_SETUP_FUNCTION},
+ {NULL, PARAM_UNRECOGNISED}
+};
+
+/*
+ * Look up a param name to find the enum value for the
+ * param, or PARAM_UNRECOGNISED if not found.
+ */
+static int
+get_param_key(const char * const param_name)
+{
+ OutputPluginParam *param = ¶m_lookup[0];
+
+ do {
+ if (strcmp(param->paramname, param_name) == 0)
+ return param->paramkey;
+ param++;
+ } while (param->paramname != NULL);
+
+ return PARAM_UNRECOGNISED;
+}
+
+
+void
+process_parameters_v1(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ ListCell *lc;
+
+ /*
+ * max_proto_version and min_proto_version are specified
+ * as required, and must be parsed before anything else.
+ *
+ * TODO: We should still parse them as optional and
+ * delay the ERROR until after the startup reply.
+ */
+ val = get_param(options, "max_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_max_proto_version = DatumGetUInt32(val);
+
+ val = get_param(options, "min_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_min_proto_version = DatumGetUInt32(val);
+
+ /* Examine all the other params in the v1 message. */
+ foreach(lc, options)
+ {
+ DefElem *elem = lfirst(lc);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Check each param, whether or not we recognise it */
+ switch(get_param_key(elem->defname))
+ {
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+
+ case PARAM_BINARY_BIGENDIAN:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_bigendian_set = true;
+ data->client_binary_bigendian = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_DATUM:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofdatum = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_INT:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofint = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_LONG:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeoflong = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_FLOAT4BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_FLOAT8BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_INTEGER_DATETIMES:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_intdatetimes_set = true;
+ data->client_binary_intdatetimes = DatumGetBool(val);
+ break;
+
+ case PARAM_EXPECTED_ENCODING:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_STRING);
+ data->client_expected_encoding = DatumGetCString(val);
+ break;
+
+ case PARAM_PG_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_pg_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_FORWARD_CHANGESETS:
+ /*
+ * Check to see if the client asked for changeset forwarding
+ *
+ * Note that we cannot support this on 9.4. We'll tell the client
+ * in the startup reply message.
+ */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_forward_changesets_set = true;
+ data->client_forward_changesets = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_WANT_INTERNAL_BASETYPES:
+ /* check if we want to use internal data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_internal_basetypes_set = true;
+ data->client_want_internal_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_WANT_BINARY_BASETYPES:
+ /* check if we want to use binary data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_binary_basetypes_set = true;
+ data->client_want_binary_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_BASETYPES_MAJOR_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_basetypes_major_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_HOOKS_SETUP_FUNCTION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_QUALIFIED_NAME);
+ data->hooks_setup_funcname = (List*) PointerGetDatum(val);
+ break;
+
+ case PARAM_UNRECOGNISED:
+ ereport(DEBUG1,
+ (errmsg("Unrecognised pglogical parameter %s ignored", elem->defname)));
+ break;
+ }
+ }
+}
+
+/*
+ * Read parameters sent by client at startup and store recognised
+ * ones in the parameters PGLogicalOutputData.
+ *
+ * The PGLogicalOutputData must have all client-surprised parameter fields
+ * zeroed, such as by memset or palloc0, since values not supplied
+ * by the client are not set.
+ */
+int
+process_parameters(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ int params_format;
+
+ val = get_param(options, "startup_params_format", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+
+ params_format = DatumGetUInt32(val);
+
+ if (params_format == 1)
+ {
+ process_parameters_v1(options, data);
+ }
+
+ return params_format;
+}
+
+static Datum
+get_param_value(DefElem *elem, bool null_ok, PGLogicalOutputParamType type)
+{
+ /* Check for NULL value */
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ {
+ if (null_ok)
+ return (Datum) 0;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("parameter \"%s\" cannot be NULL", elem->defname)));
+ }
+
+ switch (type)
+ {
+ case OUTPUT_PARAM_TYPE_UINT32:
+ return UInt32GetDatum(parse_param_uint32(elem));
+ case OUTPUT_PARAM_TYPE_BOOL:
+ return BoolGetDatum(parse_param_bool(elem));
+ case OUTPUT_PARAM_TYPE_STRING:
+ return PointerGetDatum(pstrdup(strVal(elem->arg)));
+ case OUTPUT_PARAM_TYPE_QUALIFIED_NAME:
+ return PointerGetDatum(textToQualifiedNameList(cstring_to_text(pstrdup(strVal(elem->arg)))));
+ default:
+ elog(ERROR, "unknown parameter type %d", type);
+ }
+}
+
+/*
+ * Param parsing
+ *
+ * This is not exactly fast but since it's only called on replication start
+ * we'll leave it for now.
+ */
+static Datum
+get_param(List *options, const char *name, bool missing_ok, bool null_ok,
+ PGLogicalOutputParamType type, bool *found)
+{
+ ListCell *option;
+
+ *found = false;
+
+ foreach(option, options)
+ {
+ DefElem *elem = lfirst(option);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Search until matching parameter found */
+ if (pg_strcasecmp(name, elem->defname))
+ continue;
+
+ *found = true;
+
+ return get_param_value(elem, null_ok, type);
+ }
+
+ if (!missing_ok)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("missing required parameter \"%s\"", name)));
+
+ return (Datum) 0;
+}
+
+static bool
+parse_param_bool(DefElem *elem)
+{
+ bool res;
+
+ if (!parse_bool(strVal(elem->arg), &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse boolean value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return res;
+}
+
+static uint32
+parse_param_uint32(DefElem *elem)
+{
+ int64 res;
+
+ if (!scanint8(strVal(elem->arg), true, &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse integer value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ if (res > PG_UINT32_MAX || res < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("value \"%s\" out of range for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return (uint32) res;
+}
+
+static void
+append_startup_msg_key(StringInfo si, const char *key)
+{
+ appendStringInfoString(si, key);
+ appendStringInfoChar(si, '\0');
+}
+
+static void
+append_startup_msg_s(StringInfo si, const char *key, const char *val)
+{
+ append_startup_msg_key(si, key);
+ appendStringInfoString(si, val);
+ appendStringInfoChar(si, '\0');
+}
+
+static void
+append_startup_msg_i(StringInfo si, const char *key, int val)
+{
+ append_startup_msg_key(si, key);
+ appendStringInfo(si, "%d", val);
+ appendStringInfoChar(si, '\0');
+}
+
+static void
+append_startup_msg_b(StringInfo si, const char *key, bool val)
+{
+ append_startup_msg_s(si, key, val ? "t" : "f");
+}
+
+/*
+ * This builds the protocol startup message, which is always the first
+ * message on the wire after the client sends START_REPLICATION.
+ *
+ * It confirms to the client that we could apply requested options, and
+ * tells the client our capabilities.
+ *
+ * The message is a series of null-terminated strings, alternating keys
+ * and values.
+ *
+ * See the protocol docs for details.
+ *
+ * Any additional parameters provided by the startup hook are also output
+ * now.
+ *
+ * The output param 'msg' is a null-terminated char* palloc'd in the current
+ * memory context and the length 'len' of that string that is valid. The caller
+ * should pfree the result after use.
+ *
+ * This is a bit less efficient than direct pq_sendblah calls, but
+ * separates config handling from the protocol implementation, and
+ * it's not like startup msg performance matters much.
+ */
+void
+prepare_startup_message(PGLogicalOutputData *data, char **msg, int *len)
+{
+ StringInfoData si;
+ ListCell *lc;
+
+ initStringInfo(&si);
+
+ append_startup_msg_s(&si, "max_proto_version", "1");
+ append_startup_msg_s(&si, "min_proto_version", "1");
+
+ /* We don't support understand column types yet */
+ append_startup_msg_b(&si, "coltypes", false);
+
+ /* Info about our Pg host */
+ append_startup_msg_i(&si, "pg_version_num", PG_VERSION_NUM);
+ append_startup_msg_s(&si, "pg_version", PG_VERSION);
+ append_startup_msg_i(&si, "pg_catversion", CATALOG_VERSION_NO);
+
+ append_startup_msg_s(&si, "encoding", GetDatabaseEncodingName());
+
+ append_startup_msg_b(&si, "forward_changesets",
+ data->forward_changesets);
+ append_startup_msg_b(&si, "forward_changeset_origins",
+ data->forward_changeset_origins);
+
+ /* binary options enabled */
+ append_startup_msg_b(&si, "binary.internal_basetypes",
+ data->allow_internal_basetypes);
+ append_startup_msg_b(&si, "binary.binary_basetypes",
+ data->allow_binary_basetypes);
+
+ /* Binary format characteristics of server */
+ append_startup_msg_i(&si, "binary.basetypes_major_version", PG_VERSION_NUM/100);
+ append_startup_msg_i(&si, "binary.sizeof_int", sizeof(int));
+ append_startup_msg_i(&si, "binary.sizeof_long", sizeof(long));
+ append_startup_msg_i(&si, "binary.sizeof_datum", sizeof(Datum));
+ append_startup_msg_i(&si, "binary.maxalign", MAXIMUM_ALIGNOF);
+ append_startup_msg_b(&si, "binary.bigendian", server_bigendian());
+ append_startup_msg_b(&si, "binary.float4_byval", server_float4_byval());
+ append_startup_msg_b(&si, "binary.float8_byval", server_float8_byval());
+ append_startup_msg_b(&si, "binary.integer_datetimes", server_integer_datetimes());
+ /* We don't know how to send in anything except our host's format */
+ append_startup_msg_i(&si, "binary.binary_pg_version",
+ PG_VERSION_NUM/100);
+
+
+ /*
+ * Confirm that we've enabled any requested hook functions.
+ */
+ append_startup_msg_b(&si, "hooks.startup_hook_enabled",
+ data->hooks.startup_hook != NULL);
+ append_startup_msg_b(&si, "hooks.shutdown_hook_enabled",
+ data->hooks.shutdown_hook != NULL);
+ append_startup_msg_b(&si, "hooks.row_filter_enabled",
+ data->hooks.row_filter_hook != NULL);
+ append_startup_msg_b(&si, "hooks.transaction_filter_enabled",
+ data->hooks.txn_filter_hook != NULL);
+
+ /*
+ * Output any extra params supplied by a startup hook.
+ */
+ foreach(lc, data->extra_startup_params)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ append_startup_msg_s(&si, param->defname, strVal(param->arg));
+ }
+
+ *msg = si.data;
+ *len = si.len;
+}
diff --git a/contrib/pglogical_output/pglogical_config.h b/contrib/pglogical_output/pglogical_config.h
new file mode 100644
index 0000000..fe07041
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.h
@@ -0,0 +1,56 @@
+#ifndef PG_LOGICAL_CONFIG_H
+#define PG_LOGICAL_CONFIG_H
+
+#ifndef PG_VERSION_NUM
+#error <postgres.h> must be included first
+#endif
+
+inline static bool
+server_float4_byval(void)
+{
+#ifdef USE_FLOAT4_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_float8_byval(void)
+{
+#ifdef USE_FLOAT8_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_integer_datetimes(void)
+{
+#ifdef USE_INTEGER_DATETIMES
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_bigendian(void)
+{
+#ifdef WORDS_BIGENDIAN
+ return true;
+#else
+ return false;
+#endif
+}
+
+typedef struct List List;
+typedef struct PGLogicalOutputData PGLogicalOutputData;
+
+extern int process_parameters(List *options, PGLogicalOutputData *data);
+
+extern void prepare_startup_message(PGLogicalOutputData *data,
+ char **msg, int *length);
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_hooks.c b/contrib/pglogical_output/pglogical_hooks.c
new file mode 100644
index 0000000..652d48f
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.c
@@ -0,0 +1,234 @@
+#include "postgres.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#ifdef HAVE_REPLICATION_ORIGINS
+#include "replication/origin.h"
+#endif
+
+#include "parser/parse_func.h"
+
+#include "utils/acl.h"
+#include "utils/lsyscache.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_hooks.h"
+#include "pglogical_output.h"
+
+/*
+ * Returns Oid of the hooks function specified in funcname.
+ *
+ * Error is thrown if function doesn't exist or doen't return correct datatype
+ * or is volatile.
+ */
+static Oid
+get_hooks_function_oid(List *funcname)
+{
+ Oid funcid;
+ Oid funcargtypes[1];
+
+ funcargtypes[0] = INTERNALOID;
+
+ /* find the the function */
+ funcid = LookupFuncName(funcname, 1, funcargtypes, false);
+
+ /* Validate that the function returns void */
+ if (get_func_rettype(funcid) != VOIDOID)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must return void",
+ NameListToString(funcname))));
+ }
+
+ if (func_volatile(funcid) == PROVOLATILE_VOLATILE)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must not be VOLATILE",
+ NameListToString(funcname))));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(funcname))));
+ }
+
+ return funcid;
+}
+
+/*
+ * If a hook setup function was specified in the startup parameters, look it up
+ * in the catalogs, check permissions, call it, and store the resulting hook
+ * info struct.
+ */
+void
+load_hooks(PGLogicalOutputData *data)
+{
+ Oid hooks_func;
+ MemoryContext old_ctxt;
+ bool txn_started = false;
+
+ if (!IsTransactionState())
+ {
+ txn_started = true;
+ StartTransactionCommand();
+ }
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+ hooks_func = get_hooks_function_oid(data->hooks_setup_funcname);
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) OidFunctionCall1(hooks_func, PointerGetDatum(&data->hooks));
+ MemoryContextSwitchTo(old_ctxt);
+
+ elog(DEBUG3, "pglogical_output: Loaded hooks from function %u. Hooks are: \n"
+ "\tstartup_hook: %p\n"
+ "\tshutdown_hook: %p\n"
+ "\trow_filter_hook: %p\n"
+ "\ttxn_filter_hook: %p\n"
+ "\thooks_private_data: %p\n",
+ hooks_func,
+ data->hooks.startup_hook,
+ data->hooks.shutdown_hook,
+ data->hooks.row_filter_hook,
+ data->hooks.txn_filter_hook,
+ data->hooks.hooks_private_data);
+ }
+
+ if (txn_started)
+ CommitTransactionCommand();
+}
+
+void
+call_startup_hook(PGLogicalOutputData *data, List *plugin_params)
+{
+ struct PGLogicalStartupHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.startup_hook != NULL)
+ {
+ bool tx_started = false;
+
+ args.private_data = data->hooks.hooks_private_data;
+ args.in_params = plugin_params;
+ args.out_params = NIL;
+
+ elog(DEBUG3, "calling pglogical startup hook");
+
+ if (!IsTransactionState())
+ {
+ tx_started = true;
+ StartTransactionCommand();
+ }
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.startup_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ if (tx_started)
+ CommitTransactionCommand();
+
+ data->extra_startup_params = args.out_params;
+ /* The startup hook might change the private data seg */
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical startup hook");
+ }
+}
+
+void
+call_shutdown_hook(PGLogicalOutputData *data)
+{
+ struct PGLogicalShutdownHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.shutdown_hook != NULL)
+ {
+ args.private_data = data->hooks.hooks_private_data;
+
+ elog(DEBUG3, "calling pglogical shutdown hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.shutdown_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical shutdown hook");
+ }
+}
+
+/*
+ * Decide if the individual change should be filtered out by
+ * calling a client-provided hook.
+ */
+bool
+call_row_filter_hook(PGLogicalOutputData *data, ReorderBufferTXN *txn,
+ Relation rel, ReorderBufferChange *change)
+{
+ struct PGLogicalRowFilterArgs hook_args;
+ MemoryContext old_ctxt;
+ bool ret = true;
+
+ if (data->hooks.row_filter_hook != NULL)
+ {
+ hook_args.change_type = change->action;
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.changed_rel = rel;
+
+ elog(DEBUG3, "calling pglogical row filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.row_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical row filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
+
+bool
+call_txn_filter_hook(PGLogicalOutputData *data, RepOriginId txn_origin)
+{
+ struct PGLogicalTxnFilterArgs hook_args;
+ bool ret = true;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.txn_filter_hook != NULL)
+ {
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.origin_id = txn_origin;
+
+ elog(DEBUG3, "calling pglogical txn filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.txn_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical txn filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
diff --git a/contrib/pglogical_output/pglogical_hooks.h b/contrib/pglogical_output/pglogical_hooks.h
new file mode 100644
index 0000000..df661f3
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.h
@@ -0,0 +1,22 @@
+#ifndef PGLOGICAL_HOOKS_H
+#define PGLOGICAL_HOOKS_H
+
+#include "replication/reorderbuffer.h"
+
+/* public interface for hooks */
+#include "pglogical_output/hooks.h"
+
+extern void load_hooks(PGLogicalOutputData *data);
+
+extern void call_startup_hook(PGLogicalOutputData *data, List *plugin_params);
+
+extern void call_shutdown_hook(PGLogicalOutputData *data);
+
+extern bool call_row_filter_hook(PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, Relation rel, ReorderBufferChange *change);
+
+extern bool call_txn_filter_hook(PGLogicalOutputData *data,
+ RepOriginId txn_origin);
+
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_output.c b/contrib/pglogical_output/pglogical_output.c
new file mode 100644
index 0000000..3269878
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.c
@@ -0,0 +1,463 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output/compat.h"
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+#include "pglogical_hooks.h"
+
+#include "access/hash.h"
+#include "access/sysattr.h"
+#include "access/xact.h"
+
+#include "catalog/pg_class.h"
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#include "mb/pg_wchar.h"
+
+#include "nodes/parsenodes.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/output_plugin.h"
+#include "replication/logical.h"
+#ifdef HAVE_REPLICATION_ORIGINS
+#include "replication/origin.h"
+#endif
+
+#include "utils/builtins.h"
+#include "utils/catcache.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+PG_MODULE_MAGIC;
+
+extern void _PG_output_plugin_init(OutputPluginCallbacks *cb);
+
+/* These must be available to pg_dlsym() */
+static void pg_decode_startup(LogicalDecodingContext * ctx,
+ OutputPluginOptions *opt, bool is_init);
+static void pg_decode_shutdown(LogicalDecodingContext * ctx);
+static void pg_decode_begin_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn);
+static void pg_decode_commit_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+static void pg_decode_change(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, Relation rel,
+ ReorderBufferChange *change);
+
+#ifdef HAVE_REPLICATION_ORIGINS
+static bool pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id);
+#endif
+
+static void send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message);
+
+static bool startup_message_sent = false;
+
+/* specify output plugin callbacks */
+void
+_PG_output_plugin_init(OutputPluginCallbacks *cb)
+{
+ AssertVariableIsOfType(&_PG_output_plugin_init, LogicalOutputPluginInit);
+
+ cb->startup_cb = pg_decode_startup;
+ cb->begin_cb = pg_decode_begin_txn;
+ cb->change_cb = pg_decode_change;
+ cb->commit_cb = pg_decode_commit_txn;
+#ifdef HAVE_REPLICATION_ORIGINS
+ cb->filter_by_origin_cb = pg_decode_origin_filter;
+#endif
+ cb->shutdown_cb = pg_decode_shutdown;
+}
+
+static bool
+check_binary_compatibility(PGLogicalOutputData *data)
+{
+ if (data->client_binary_basetypes_major_version != PG_VERSION_NUM / 100)
+ return false;
+
+ if (data->client_binary_bigendian_set
+ && data->client_binary_bigendian != server_bigendian())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian mis-match");
+ return false;
+ }
+
+ if (data->client_binary_sizeofdatum != 0
+ && data->client_binary_sizeofdatum != sizeof(Datum))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(Datum) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeofint != 0
+ && data->client_binary_sizeofint != sizeof(int))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(int) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeoflong != 0
+ && data->client_binary_sizeoflong != sizeof(long))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(long) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float4byval_set
+ && data->client_binary_float4byval != server_float4_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float4byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float8byval_set
+ && data->client_binary_float8byval != server_float8_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float8byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_intdatetimes_set
+ && data->client_binary_intdatetimes != server_integer_datetimes())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian integer datetimes mismatch");
+ return false;
+ }
+
+ return true;
+}
+
+/* initialize this plugin */
+static void
+pg_decode_startup(LogicalDecodingContext * ctx, OutputPluginOptions *opt,
+ bool is_init)
+{
+ PGLogicalOutputData *data = palloc0(sizeof(PGLogicalOutputData));
+
+ data->context = AllocSetContextCreate(TopMemoryContext,
+ "pglogical conversion context",
+ ALLOCSET_DEFAULT_MINSIZE,
+ ALLOCSET_DEFAULT_INITSIZE,
+ ALLOCSET_DEFAULT_MAXSIZE);
+ data->allow_internal_basetypes = false;
+ data->allow_binary_basetypes = false;
+
+ ctx->output_plugin_private = data;
+
+ /*
+ * Tell logical decoding that we will be doing binary output. This is
+ * not the same thing as the selection of binary or text format for
+ * output of individual fields.
+ */
+ opt->output_type = OUTPUT_PLUGIN_BINARY_OUTPUT;
+
+ /*
+ * This is replication start and not slot initialization.
+ *
+ * Parse and validate options passed by the client.
+ */
+ if (!is_init)
+ {
+ int params_format;
+
+ startup_message_sent = false;
+
+ /* Now parse the rest of the params and ERROR if we see any we don't recognise */
+ params_format = process_parameters(ctx->output_plugin_options, data);
+
+ /* TODO: delay until after sending startup reply */
+ if (params_format != 1)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent startup parameters in format %d but we only support format 1",
+ params_format)));
+
+ /* TODO: Should delay our ERROR until sending startup reply */
+ if (data->client_min_proto_version > PG_LOGICAL_PROTO_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent min_proto_version=%d but we only support protocol %d or lower",
+ data->client_min_proto_version, PG_LOGICAL_PROTO_VERSION_NUM)));
+
+ /* TODO: Should delay our ERROR until sending startup reply */
+ if (data->client_max_proto_version < PG_LOGICAL_PROTO_MIN_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent max_proto_version=%d but we only support protocol %d or higher",
+ data->client_max_proto_version, PG_LOGICAL_PROTO_MIN_VERSION_NUM)));
+
+ /* check for encoding match if specific encoding demanded by client */
+ /* TODO: Should parse encoding name and compare properly */
+ if (data->client_expected_encoding != NULL
+ && strlen(data->client_expected_encoding) != 0
+ && strcmp(data->client_expected_encoding, GetDatabaseEncodingName()) != 0)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("only \"%s\" encoding is supported by this server, client requested %s",
+ GetDatabaseEncodingName(), data->client_expected_encoding)));
+ }
+
+ if (data->client_want_internal_basetypes)
+ {
+ data->allow_internal_basetypes =
+ check_binary_compatibility(data);
+ }
+
+ if (data->client_want_binary_basetypes &&
+ data->client_binary_basetypes_major_version == PG_VERSION_NUM / 100)
+ {
+ data->allow_binary_basetypes = true;
+ }
+
+ /*
+ * Will we forward changesets? We have to if we're on 9.4;
+ * otherwise honour the client's request.
+ */
+ if (PG_VERSION_NUM/100 == 904)
+ {
+ /*
+ * 9.4 unconditionally forwards changesets due to lack of
+ * replication origins, and it can't ever send origin info
+ * for the same reason.
+ */
+ data->forward_changesets = true;
+ data->forward_changeset_origins = false;
+
+ if (data->client_forward_changesets_set
+ && !data->client_forward_changesets)
+ {
+ ereport(DEBUG1,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("Cannot disable changeset forwarding on PostgreSQL 9.4")));
+ }
+ }
+ else if (data->client_forward_changesets_set
+ && data->client_forward_changesets)
+ {
+ /* Client explicitly asked for forwarding; forward csets and origins */
+ data->forward_changesets = true;
+ data->forward_changeset_origins = true;
+ }
+ else
+ {
+ /* Default to not forwarding or honour client's request not to fwd */
+ data->forward_changesets = false;
+ data->forward_changeset_origins = false;
+ }
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+
+ data->hooks_mctxt = AllocSetContextCreate(ctx->context,
+ "pglogical_output hooks context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+ load_hooks(data);
+ call_startup_hook(data, ctx->output_plugin_options);
+ }
+ }
+}
+
+/*
+ * BEGIN callback
+ */
+void
+pg_decode_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+ bool send_replication_origin = data->forward_changeset_origins;
+
+ if (!startup_message_sent)
+ send_startup_message( ctx, data, false /* can't be last message */);
+
+#ifdef HAVE_REPLICATION_ORIGINS
+ /* If the record didn't originate locally, send origin info */
+ send_replication_origin &= txn->origin_id != InvalidRepOriginId;
+#endif
+
+ OutputPluginPrepareWrite(ctx, !send_replication_origin);
+ pglogical_write_begin(ctx->out, txn);
+
+#ifdef HAVE_REPLICATION_ORIGINS
+ if (send_replication_origin)
+ {
+ char *origin;
+
+ /* Message boundary */
+ OutputPluginWrite(ctx, false);
+ OutputPluginPrepareWrite(ctx, true);
+
+ /*
+ * XXX: which behaviour we want here?
+ *
+ * Alternatives:
+ * - don't send origin message if origin name not found
+ * (that's what we do now)
+ * - throw error - that will break replication, not good
+ * - send some special "unknown" origin
+ */
+ if (replorigin_by_oid(txn->origin_id, true, &origin))
+ pglogical_write_origin(ctx->out, origin, txn->origin_lsn);
+ }
+#endif
+
+ OutputPluginWrite(ctx, true);
+}
+
+/*
+ * COMMIT callback
+ */
+void
+pg_decode_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ OutputPluginPrepareWrite(ctx, true);
+ pglogical_write_commit(ctx->out, txn, commit_lsn);
+ OutputPluginWrite(ctx, true);
+}
+
+void
+pg_decode_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ Relation relation, ReorderBufferChange *change)
+{
+ PGLogicalOutputData *data;
+ MemoryContext old;
+
+ data = ctx->output_plugin_private;
+
+ /* First check the table filter */
+ if (!call_row_filter_hook(data, txn, relation, change))
+ return;
+
+ /* Avoid leaking memory by using and resetting our own context */
+ old = MemoryContextSwitchTo(data->context);
+
+ /* TODO: add caching (send only if changed) */
+ OutputPluginPrepareWrite(ctx, false);
+ pglogical_write_rel(ctx->out, relation);
+ OutputPluginWrite(ctx, false);
+
+ /* Send the data */
+ switch (change->action)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ OutputPluginPrepareWrite(ctx, true);
+ pglogical_write_insert(ctx->out, data, relation,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ {
+ HeapTuple oldtuple = change->data.tp.oldtuple ?
+ &change->data.tp.oldtuple->tuple : NULL;
+
+ OutputPluginPrepareWrite(ctx, true);
+ pglogical_write_update(ctx->out, data, relation, oldtuple,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ }
+ case REORDER_BUFFER_CHANGE_DELETE:
+ if (change->data.tp.oldtuple)
+ {
+ OutputPluginPrepareWrite(ctx, true);
+ pglogical_write_delete(ctx->out, data, relation,
+ &change->data.tp.oldtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ }
+ else
+ elog(DEBUG1, "didn't send DELETE change because of missing oldtuple");
+ break;
+ default:
+ Assert(false);
+ }
+
+ /* Cleanup */
+ MemoryContextSwitchTo(old);
+ MemoryContextReset(data->context);
+}
+
+#ifdef HAVE_REPLICATION_ORIGINS
+/*
+ * Decide if the whole transaction with specific origin should be filtered out.
+ */
+static bool
+pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id)
+{
+ PGLogicalOutputData *data = ctx->output_plugin_private;
+
+ if (!call_txn_filter_hook(data, origin_id))
+ return true;
+
+ if (!data->forward_changesets && origin_id != InvalidRepOriginId)
+ return true;
+
+ return false;
+}
+#endif
+
+static void
+send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message)
+{
+ char *msg;
+ int len;
+
+ Assert(!startup_message_sent);
+
+ prepare_startup_message(data, &msg, &len);
+
+ /*
+ * We could free the extra_startup_params DefElem list here, but it's
+ * pretty harmless to just ignore it, since it's in the decoding memory
+ * context anyway, and we don't know if it's safe to free the defnames or
+ * not.
+ */
+
+ OutputPluginPrepareWrite(ctx, last_message);
+ write_startup_message(ctx->out, msg, len);
+ OutputPluginWrite(ctx, last_message);
+
+ pfree(msg);
+
+ startup_message_sent = true;
+}
+
+static void pg_decode_shutdown(LogicalDecodingContext * ctx)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+
+ call_shutdown_hook(data);
+
+ if (data->hooks_mctxt != NULL)
+ {
+ MemoryContextDelete(data->hooks_mctxt);
+ data->hooks_mctxt = NULL;
+ }
+}
diff --git a/contrib/pglogical_output/pglogical_output.h b/contrib/pglogical_output/pglogical_output.h
new file mode 100644
index 0000000..e835dca
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.h
@@ -0,0 +1,98 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.h
+ * pglogical output plugin
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_OUTPUT_H
+#define PG_LOGICAL_OUTPUT_H
+
+#include "nodes/parsenodes.h"
+
+#include "replication/logical.h"
+#include "replication/output_plugin.h"
+
+#include "storage/lock.h"
+
+#include "pglogical_output/hooks.h"
+
+#define PG_LOGICAL_PROTO_VERSION_NUM 1
+#define PG_LOGICAL_PROTO_MIN_VERSION_NUM 1
+
+/*
+ * The name of a hook function. This is used instead of the usual List*
+ * because can serve as a hash key.
+ *
+ * Must be zeroed on allocation if used as a hash key since padding is
+ * *not* ignored on compare.
+ */
+typedef struct HookFuncName
+{
+ /* funcname is more likely to be unique, so goes first */
+ char function[NAMEDATALEN];
+ char schema[NAMEDATALEN];
+} HookFuncName;
+
+typedef struct PGLogicalOutputData
+{
+ MemoryContext context;
+
+ /* protocol */
+ bool allow_internal_basetypes;
+ bool allow_binary_basetypes;
+ bool forward_changesets;
+ bool forward_changeset_origins;
+
+ /*
+ * client info
+ *
+ * TODO: Lots of this should move to a separate
+ * shorter-lived struct used only during parameter
+ * reading.
+ */
+ uint32 client_pg_version;
+ uint32 client_max_proto_version;
+ uint32 client_min_proto_version;
+ const char *client_expected_encoding;
+ uint32 client_binary_basetypes_major_version;
+ bool client_want_internal_basetypes_set;
+ bool client_want_internal_basetypes;
+ bool client_want_binary_basetypes_set;
+ bool client_want_binary_basetypes;
+ bool client_binary_bigendian_set;
+ bool client_binary_bigendian;
+ uint32 client_binary_sizeofdatum;
+ uint32 client_binary_sizeofint;
+ uint32 client_binary_sizeoflong;
+ bool client_binary_float4byval_set;
+ bool client_binary_float4byval;
+ bool client_binary_float8byval_set;
+ bool client_binary_float8byval;
+ bool client_binary_intdatetimes_set;
+ bool client_binary_intdatetimes;
+ bool client_forward_changesets_set;
+ bool client_forward_changesets;
+
+ /* hooks */
+ List *hooks_setup_funcname;
+ struct PGLogicalHooks hooks;
+ MemoryContext hooks_mctxt;
+
+ /* DefElem<String> list populated by startup hook */
+ List *extra_startup_params;
+} PGLogicalOutputData;
+
+typedef struct PGLogicalTupleData
+{
+ Datum values[MaxTupleAttributeNumber];
+ bool nulls[MaxTupleAttributeNumber];
+ bool changed[MaxTupleAttributeNumber];
+} PGLogicalTupleData;
+
+#endif /* PG_LOGICAL_OUTPUT_H */
diff --git a/contrib/pglogical_output/pglogical_output/README b/contrib/pglogical_output/pglogical_output/README
new file mode 100644
index 0000000..5480e5c
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/README
@@ -0,0 +1,7 @@
+/*
+ * This directory contains the public header files for the pglogical_output
+ * extension. It is installed into the PostgreSQL source tree when the extension
+ * is installed.
+ *
+ * These headers are not part of the PostgreSQL project its self.
+ */
diff --git a/contrib/pglogical_output/pglogical_output/compat.h b/contrib/pglogical_output/pglogical_output/compat.h
new file mode 100644
index 0000000..6d0b778
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/compat.h
@@ -0,0 +1,19 @@
+#ifndef PG_LOGICAL_COMPAT_H
+#define PG_LOGICAL_COMPAT_H
+
+#include "pg_config.h"
+
+/* 9.4 lacks replication origins */
+#if PG_VERSION_NUM >= 90500
+#define HAVE_REPLICATION_ORIGINS
+#else
+/* To allow the same signature on hooks in 9.4 */
+typedef uint16 RepOriginId;
+#endif
+
+/* 9.4 lacks PG_UINT32_MAX */
+#ifndef PG_UINT32_MAX
+#define PG_UINT32_MAX UINT32_MAX
+#endif
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_output/hooks.h b/contrib/pglogical_output/pglogical_output/hooks.h
new file mode 100644
index 0000000..139af44
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/hooks.h
@@ -0,0 +1,74 @@
+#ifndef PGLOGICAL_OUTPUT_HOOKS_H
+#define PGLOGICAL_OUTPUT_HOOKS_H
+
+#include "access/xlogdefs.h"
+#include "nodes/pg_list.h"
+#include "utils/rel.h"
+#include "utils/palloc.h"
+#include "replication/reorderbuffer.h"
+
+#include "pglogical_output/compat.h"
+
+struct PGLogicalOutputData;
+typedef struct PGLogicalOutputData PGLogicalOutputData;
+
+/*
+ * This header is to be included by extensions that implement pglogical output
+ * plugin callback hooks for transaction origin and row filtering, etc. It is
+ * installed as "pglogical_output/hooks.h"
+ *
+ * See the README.md and the example in examples/hooks/ for details on hooks.
+ */
+
+
+struct PGLogicalStartupHookArgs
+{
+ void *private_data;
+ List *in_params;
+ List *out_params;
+};
+
+typedef void (*pglogical_startup_hook_fn)(struct PGLogicalStartupHookArgs *args);
+
+
+struct PGLogicalTxnFilterArgs
+{
+ void *private_data;
+ RepOriginId origin_id;
+};
+
+typedef bool (*pglogical_txn_filter_hook_fn)(struct PGLogicalTxnFilterArgs *args);
+
+
+struct PGLogicalRowFilterArgs
+{
+ void *private_data;
+ Relation changed_rel;
+ enum ReorderBufferChangeType change_type;
+};
+
+typedef bool (*pglogical_row_filter_hook_fn)(struct PGLogicalRowFilterArgs *args);
+
+
+struct PGLogicalShutdownHookArgs
+{
+ void *private_data;
+};
+
+typedef void (*pglogical_shutdown_hook_fn)(struct PGLogicalShutdownHookArgs *args);
+
+/*
+ * This struct is passed to the pglogical_get_hooks_fn as the first argument,
+ * typed 'internal', and is unwrapped with `DatumGetPointer`.
+ */
+struct PGLogicalHooks
+{
+ pglogical_startup_hook_fn startup_hook;
+ pglogical_shutdown_hook_fn shutdown_hook;
+ pglogical_txn_filter_hook_fn txn_filter_hook;
+ pglogical_row_filter_hook_fn row_filter_hook;
+ void *hooks_private_data;
+};
+
+
+#endif /* PGLOGICAL_OUTPUT_HOOKS_H */
diff --git a/contrib/pglogical_output/pglogical_proto.c b/contrib/pglogical_output/pglogical_proto.c
new file mode 100644
index 0000000..0ce3014
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.c
@@ -0,0 +1,484 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.c
+ * pglogical protocol functions
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+
+#include "access/sysattr.h"
+#include "access/tuptoaster.h"
+#include "access/xact.h"
+
+#include "catalog/catversion.h"
+#include "catalog/index.h"
+
+#include "catalog/namespace.h"
+#include "catalog/pg_class.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_namespace.h"
+#include "catalog/pg_type.h"
+
+#include "commands/dbcommands.h"
+
+#include "executor/spi.h"
+
+#include "libpq/pqformat.h"
+
+#include "mb/pg_wchar.h"
+
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/typcache.h"
+
+#define IS_REPLICA_IDENTITY 1
+
+static void pglogical_write_attrs(StringInfo out, Relation rel);
+static void pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple);
+static char decide_datum_transfer(Form_pg_attribute att,
+ Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes);
+
+/*
+ * Write relation description to the output stream.
+ */
+void
+pglogical_write_rel(StringInfo out, Relation rel)
+{
+ const char *nspname;
+ uint8 nspnamelen;
+ const char *relname;
+ uint8 relnamelen;
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'R'); /* sending RELATION */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ nspname = get_namespace_name(rel->rd_rel->relnamespace);
+ if (nspname == NULL)
+ elog(ERROR, "cache lookup failed for namespace %u",
+ rel->rd_rel->relnamespace);
+ nspnamelen = strlen(nspname) + 1;
+
+ relname = NameStr(rel->rd_rel->relname);
+ relnamelen = strlen(relname) + 1;
+
+ pq_sendbyte(out, nspnamelen); /* schema name length */
+ pq_sendbytes(out, nspname, nspnamelen);
+
+ pq_sendbyte(out, relnamelen); /* table name length */
+ pq_sendbytes(out, relname, relnamelen);
+
+ /* send the attribute info */
+ pglogical_write_attrs(out, rel);
+}
+
+/*
+ * Write relation attributes to the outputstream.
+ */
+static void
+pglogical_write_attrs(StringInfo out, Relation rel)
+{
+ TupleDesc desc;
+ int i;
+ uint16 nliveatts = 0;
+ Bitmapset *idattrs;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'A'); /* sending ATTRS */
+
+ /* send number of live attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* fetch bitmap of REPLICATION IDENTITY attributes */
+ idattrs = RelationGetIndexAttrBitmap(rel, INDEX_ATTR_BITMAP_IDENTITY_KEY);
+
+ /* send the attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ Form_pg_attribute att = desc->attrs[i];
+ uint8 flags = 0;
+ uint16 len;
+ const char *attname;
+
+ if (att->attisdropped)
+ continue;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ idattrs))
+ flags |= IS_REPLICA_IDENTITY;
+
+ pq_sendbyte(out, 'C'); /* column definition follows */
+ pq_sendbyte(out, flags);
+
+ pq_sendbyte(out, 'N'); /* column name block follows */
+ attname = NameStr(att->attname);
+ len = strlen(attname) + 1;
+ pq_sendint(out, len, 2);
+ pq_sendbytes(out, attname, len); /* data */
+ }
+}
+
+/*
+ * Write BEGIN to the output stream.
+ */
+void
+pglogical_write_begin(StringInfo out, ReorderBufferTXN *txn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'B'); /* BEGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, txn->final_lsn);
+ pq_sendint64(out, txn->commit_time);
+ pq_sendint(out, txn->xid, 4);
+}
+
+/*
+ * Write COMMIT to the output stream.
+ */
+void
+pglogical_write_commit(StringInfo out, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'C'); /* sending COMMIT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* send fixed fields */
+ pq_sendint64(out, commit_lsn);
+ pq_sendint64(out, txn->end_lsn);
+ pq_sendint64(out, txn->commit_time);
+}
+
+/*
+ * Write ORIGIN to the output stream.
+ */
+void
+pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn)
+{
+ uint8 flags = 0;
+ uint8 len;
+
+ Assert(strlen(origin) < 255);
+
+ pq_sendbyte(out, 'O'); /* ORIGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, origin_lsn);
+
+ /* origin */
+ len = strlen(origin) + 1;
+ pq_sendbyte(out, len);
+ pq_sendbytes(out, origin, len);
+}
+
+/*
+ * Write INSERT to the output stream.
+ */
+void
+pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'I'); /* action INSERT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write UPDATE to the output stream.
+ */
+void
+pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'U'); /* action UPDATE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ if (oldtuple != NULL)
+ {
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+ }
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write DELETE to the output stream.
+ */
+void
+pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'D'); /* action DELETE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+}
+
+/*
+ * Most of the brains for startup message creation lives in
+ * pglogical_config.c, so this presently just sends the set of key/value pairs.
+ */
+void
+write_startup_message(StringInfo out, const char *msg, int len)
+{
+ pq_sendbyte(out, 'S'); /* message type field */
+ pq_sendbyte(out, 1); /* startup message version */
+ pq_sendbytes(out, msg, len); /* null-terminated key/value pairs */
+}
+
+/*
+ * Write a tuple to the outputstream, in the most efficient format possible.
+ */
+static void
+pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple)
+{
+ TupleDesc desc;
+ Datum values[MaxTupleAttributeNumber];
+ bool isnull[MaxTupleAttributeNumber];
+ int i;
+ uint16 nliveatts = 0;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'T'); /* sending TUPLE */
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* try to allocate enough memory from the get go */
+ enlargeStringInfo(out, tuple->t_len +
+ nliveatts * (1 + 4));
+
+ /*
+ * XXX: should this prove to be a relevant bottleneck, it might be
+ * interesting to inline heap_deform_tuple() here, we don't actually need
+ * the information in the form we get from it.
+ */
+ heap_deform_tuple(tuple, desc, values, isnull);
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ HeapTuple typtup;
+ Form_pg_type typclass;
+ Form_pg_attribute att = desc->attrs[i];
+ char transfer_type;
+
+ /* skip dropped columns */
+ if (att->attisdropped)
+ continue;
+
+ if (isnull[i])
+ {
+ pq_sendbyte(out, 'n'); /* null column */
+ continue;
+ }
+ else if (att->attlen == -1 && VARATT_IS_EXTERNAL_ONDISK(values[i]))
+ {
+ pq_sendbyte(out, 'u'); /* unchanged toast column */
+ continue;
+ }
+
+ typtup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid));
+ if (!HeapTupleIsValid(typtup))
+ elog(ERROR, "cache lookup failed for type %u", att->atttypid);
+ typclass = (Form_pg_type) GETSTRUCT(typtup);
+
+ transfer_type = decide_datum_transfer(att, typclass,
+ data->allow_internal_basetypes,
+ data->allow_binary_basetypes);
+
+ switch (transfer_type)
+ {
+ case 'i':
+ pq_sendbyte(out, 'i'); /* internal-format binary data follows */
+
+ /* pass by value */
+ if (att->attbyval)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ enlargeStringInfo(out, att->attlen);
+ store_att_byval(out->data + out->len, values[i],
+ att->attlen);
+ out->len += att->attlen;
+ out->data[out->len] = '\0';
+ }
+ /* fixed length non-varlena pass-by-reference type */
+ else if (att->attlen > 0)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ appendBinaryStringInfo(out, DatumGetPointer(values[i]),
+ att->attlen);
+ }
+ /* varlena type */
+ else if (att->attlen == -1)
+ {
+ char *data = DatumGetPointer(values[i]);
+
+ /* send indirect datums inline */
+ if (VARATT_IS_EXTERNAL_INDIRECT(values[i]))
+ {
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, data);
+ data = (char *) redirect.pointer;
+ }
+
+ Assert(!VARATT_IS_EXTERNAL(data));
+
+ pq_sendint(out, VARSIZE_ANY(data), 4); /* length */
+
+ appendBinaryStringInfo(out, data, VARSIZE_ANY(data));
+ }
+ else
+ elog(ERROR, "unsupported tuple type");
+
+ break;
+
+ case 'b':
+ {
+ bytea *outputbytes;
+ int len;
+
+ pq_sendbyte(out, 'b'); /* binary send/recv data follows */
+
+ outputbytes = OidSendFunctionCall(typclass->typsend,
+ values[i]);
+
+ len = VARSIZE(outputbytes) - VARHDRSZ;
+ pq_sendint(out, len, 4); /* length */
+ pq_sendbytes(out, VARDATA(outputbytes), len); /* data */
+ pfree(outputbytes);
+ }
+ break;
+
+ default:
+ {
+ char *outputstr;
+ int len;
+
+ pq_sendbyte(out, 't'); /* 'text' data follows */
+
+ outputstr = OidOutputFunctionCall(typclass->typoutput,
+ values[i]);
+ len = strlen(outputstr) + 1;
+ pq_sendint(out, len, 4); /* length */
+ appendBinaryStringInfo(out, outputstr, len); /* data */
+ pfree(outputstr);
+ }
+ }
+
+ ReleaseSysCache(typtup);
+ }
+}
+
+/*
+ * Make the executive decision about which protocol to use.
+ */
+static char
+decide_datum_transfer(Form_pg_attribute att, Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes)
+{
+ /*
+ * Use the binary protocol, if allowed, for builtin & plain datatypes.
+ */
+ if (allow_internal_basetypes &&
+ typclass->typtype == 'b' &&
+ att->atttypid < FirstNormalObjectId &&
+ typclass->typelem == InvalidOid)
+ {
+ return 'i';
+ }
+ /*
+ * Use send/recv, if allowed, if the type is plain or builtin.
+ *
+ * XXX: we can't use send/recv for array or composite types for now due to
+ * the embedded oids.
+ */
+ else if (allow_binary_basetypes &&
+ OidIsValid(typclass->typreceive) &&
+ (att->atttypid < FirstNormalObjectId || typclass->typtype != 'c') &&
+ (att->atttypid < FirstNormalObjectId || typclass->typelem == InvalidOid))
+ {
+ return 'b';
+ }
+
+ return 't';
+}
diff --git a/contrib/pglogical_output/pglogical_proto.h b/contrib/pglogical_output/pglogical_proto.h
new file mode 100644
index 0000000..77cff87
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.h
@@ -0,0 +1,36 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.c
+ * pglogical protocol
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_H
+#define PG_LOGICAL_PROTO_H
+
+
+void pglogical_write_rel(StringInfo out, Relation rel);
+
+void pglogical_write_begin(StringInfo out, ReorderBufferTXN *txn);
+void pglogical_write_commit(StringInfo out, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn);
+
+void pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn);
+
+void pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+void pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+void pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+void write_startup_message(StringInfo out, const char *msg, int len);
+
+#endif /* PG_LOGICAL_PROTO_H */
diff --git a/contrib/pglogical_output/regression.conf b/contrib/pglogical_output/regression.conf
new file mode 100644
index 0000000..367f706
--- /dev/null
+++ b/contrib/pglogical_output/regression.conf
@@ -0,0 +1,2 @@
+wal_level = logical
+max_replication_slots = 4
diff --git a/contrib/pglogical_output/sql/basic.sql b/contrib/pglogical_output/sql/basic.sql
new file mode 100644
index 0000000..8f08bdc
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic.sql
@@ -0,0 +1,49 @@
+SET synchronous_commit = on;
+
+-- Schema setup
+
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Queue up some work to decode with a variety of types
+
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE demo;
diff --git a/contrib/pglogical_output/sql/hooks.sql b/contrib/pglogical_output/sql/hooks.sql
new file mode 100644
index 0000000..adcdfb1
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks.sql
@@ -0,0 +1,86 @@
+CREATE EXTENSION pglogical_output_plhooks;
+
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/sql/params.sql b/contrib/pglogical_output/sql/params.sql
new file mode 100644
index 0000000..24ac347
--- /dev/null
+++ b/contrib/pglogical_output/sql/params.sql
@@ -0,0 +1,77 @@
+SET synchronous_commit = on;
+
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+--
+-- Various invalid parameter combos:
+--
+
+-- Text mode is not supported
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- Currently we're sensitive to the encoding name's format (TODO)
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
diff --git a/contrib/pglogical_output/test/Makefile b/contrib/pglogical_output/test/Makefile
new file mode 100644
index 0000000..7a6c0ed
--- /dev/null
+++ b/contrib/pglogical_output/test/Makefile
@@ -0,0 +1,6 @@
+all: check
+
+check:
+ @python -m unittest discover
+
+.PHONY: check all
diff --git a/contrib/pglogical_output/test/README.md b/contrib/pglogical_output/test/README.md
new file mode 100644
index 0000000..9a91e1e
--- /dev/null
+++ b/contrib/pglogical_output/test/README.md
@@ -0,0 +1,91 @@
+What are these tests?
+---
+
+These tests exersise the pglogical protocol, parameter validation, hooks and
+filters, and the overall behaviour of the extension. They are *not* the tests
+run by `make check` or `make installcheck` on the top level source directory;
+those are the `pg_regress` tests discussed in the "tests" section of the
+top-level `README.md`.
+
+QUICK START
+---
+
+To run these tests:
+
+* Install the output plugin into your PostgreSQL instance, e.g.
+
+ make USE_PGXS=1 install
+
+ Use the same options, environment variables, etc as used for compiling,
+ most notably the `PATH` to ensure the same `pg_config` is used.
+
+* Create a temporary PostgreSQL datadir at any location of your choosing:
+
+ initdb -A trust -D tmp_install
+
+* Start the temporary PostgreSQL instance with:
+
+ PGPORT=5142 postgres -D tmp_install -c max_replication_slots=5 -c wal_level=logical -c max_wal_senders=10 -c track_commit_timestamp=on
+
+ (leave out `track_commit_timestamp=on` for 9.4)
+
+* In another session, in the test directory:
+
+ PGPORT=5142 make
+
+RUNNING JUST ONE TEST
+---
+
+To run just one test, specify the class-qualified method name.
+
+ PGPORT=5142 python test/test_filter.py FilterTest.test_filter
+
+WALSENDER VS SQL MODE
+---
+
+By default the tests use the SQL interface for logical decoding.
+
+You can instead use the walsender interface, i.e. the streaming replication
+protocol. However, this requires a patched psycopg2 at time of writing. You
+can get the branch from https://github.com/zalando/psycopg2/tree/feature/replication-protocol
+
+You should uninstall your existing `psycopg2` packages, then:
+
+ git clone https://github.com/zalando/psycopg2.git
+ git checkout feature/replication-protocol
+ PATH=/path/to/pg/bin:$PATH python setup.py build
+ sudo PATH=/path/to/pg/bin:$PATH python setup.py install
+
+Now run the tests with the extra enviroment variable PGLOGICALTEST_USEWALSENDER=1
+set, e.g.
+
+ PGLOGICALTEST_USEWALSENDER=1 PGPORT=5142 make
+
+At time of writing the walsender tests may not always be passing, as the
+SQL tests are the authorative ones.
+
+DETAILED LOGGING
+---
+
+You can get more detailed info about what's being done by setting the env var
+`PGLOGICALTEST_LOGLEVEL=DEBUG`
+
+TROUBLESHOOTING
+---
+
+No module named psycopg2
+===
+
+If you get an error like:
+
+ ImportError: No module named psycopg2
+
+you need to install `psycopg2` for your local Python install. It'll be
+available as a package via the same channel you installed Python its self from.
+
+could not access file "pglogical_output": No such file or directory
+===
+
+You forgot to install the output plugin before running the tests, or
+the tests are connecting to a different PostgreSQL instance than the
+one you installed the plugin in.
diff --git a/contrib/pglogical_output/test/base.py b/contrib/pglogical_output/test/base.py
new file mode 100644
index 0000000..ab7fec0
--- /dev/null
+++ b/contrib/pglogical_output/test/base.py
@@ -0,0 +1,285 @@
+import unittest
+import psycopg2
+import psycopg2.extras;
+import cStringIO
+import logging
+import pprint
+import psycopg2.extensions
+import select
+import time
+import sys
+import os
+from pglogical_protoreader import ProtocolReader
+
+from pglogical_proto import ReplicationMessage
+
+SLOT_NAME = 'test'
+
+class BaseDecodingInterface(object):
+ """Helper for handling the different decoding interfaces"""
+
+ conn = None
+ cur = None
+
+ def __init__(self, connstring, logger):
+ # Establish base connection, which we use in walsender mode too
+ self.logger = logger
+ self.connstring = connstring
+ self.conn = psycopg2.connect(self.connstring)
+ self.logger.debug("Acquired connection with pid %s", self.conn.get_backend_pid())
+ self.conn.autocommit = True
+ self.cur = self.conn.cursor()
+
+ def slot_exists(self):
+ self.cur.execute("SELECT 1 FROM pg_replication_slots WHERE slot_name = %s", (SLOT_NAME,))
+ return self.cur.rowcount == 1
+
+ def drop_slot_when_inactive(self):
+ self.logger.debug("Dropping slot %s", SLOT_NAME)
+ try:
+ # We can't use the walsender protocol connection to drop
+ # the slot because we have no way to exit COPY BOTH mode
+ # so close the connection (above) and drop from SQL.
+ if self.cur is not None:
+ # There's a race between walsender disconnect and the slot becoming
+ # free. We should use a DO block, but this will do for now.
+ #
+ # this is only an issue in walsender mode, but might as well do
+ # it anyway.
+ self.cur.execute("""
+ DO
+ LANGUAGE plpgsql
+ $$
+ DECLARE
+ timeleft float := 5.0;
+ _slotname name := %s;
+ BEGIN
+ IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = _slotname)
+ THEN
+ RETURN;
+ END IF;
+ WHILE (SELECT active FROM pg_catalog.pg_replication_slots WHERE slot_name = _slotname) AND timeleft > 0
+ LOOP
+ PERFORM pg_sleep(0.1);
+ timeleft := timeleft - 0.1;
+ END LOOP;
+ IF timeleft > 0 THEN
+ PERFORM pg_drop_replication_slot(_slotname);
+ ELSE
+ RAISE EXCEPTION 'Timed out waiting for slot to become unused';
+ END IF;
+ END;
+ $$
+ """, (SLOT_NAME,))
+ except psycopg2.ProgrammingError, ex:
+ self.logger.exception("Attempt to DROP slot %s failed", SLOT_NAME)
+ self.logger.debug("Dropped slot %s", SLOT_NAME)
+
+ def cleanup(self):
+ if self.cur is not None:
+ self.cur.close()
+ if self.conn is not None:
+ self.conn.close()
+
+ def _get_changes_params(self, kwargs):
+ params_dict = {
+ 'expected_encoding': 'UTF8',
+ 'min_proto_version': '1',
+ 'max_proto_version': '1',
+ 'startup_params_format': '1'
+ }
+ params_dict.update(kwargs)
+ return params_dict
+
+
+
+class SQLDecodingInterface(BaseDecodingInterface):
+ """Use the SQL level logical decoding interfaces"""
+
+ def __init__(self, connstring, parentlogger=logging.getLogger('base')):
+ BaseDecodingInterface.__init__(self, connstring, logger=parentlogger.getChild('sqldecoding:%s' % hex(id(self))))
+
+ # cleanup old slot
+ if self.slot_exists():
+ self.cur.execute("SELECT * FROM pg_drop_replication_slot(%s)", (SLOT_NAME,))
+
+ # Create slot to use in testing
+ self.cur.execute("SELECT * FROM pg_create_logical_replication_slot(%s, 'pglogical_output')", (SLOT_NAME,))
+
+ def cleanup(self):
+ self.logger.debug("Closing sql decoding connection")
+ self.drop_slot_when_inactive()
+ BaseDecodingInterface.cleanup(self)
+ self.logger.debug("Closed sql decoding connection")
+
+ def get_changes(self, kwargs = {}):
+ params_dict = self._get_changes_params(kwargs)
+
+ # Filter out entries with value None
+ params = [i for k, v in params_dict.items() for i in [k,str(v)] if v is not None]
+ # and create the slot
+ try:
+ self.cur.execute("SELECT * FROM pg_logical_slot_get_binary_changes(%s, NULL, NULL" + (", %s" * len(params)) + ")",
+ [SLOT_NAME] + params);
+ finally:
+ self.conn.commit()
+
+ for row in self.cur:
+ yield ReplicationMessage(row)
+
+
+
+class WalsenderDecodingInterface(BaseDecodingInterface):
+ """Use the replication protocol interfaces"""
+
+ walcur = None
+ walconn = None
+ select_timeout = 1
+ replication_started = False
+
+ def __init__(self, connstring, parentlogger=logging.getLogger('base')):
+ BaseDecodingInterface.__init__(self, connstring, logger=parentlogger.getChild('waldecoding:%s' % hex(id(self))))
+
+ # Establish an async logical replication connection
+ self.walconn = psycopg2.connect(self.connstring,
+ connection_factory=psycopg2.extras.LogicalReplicationConnection)
+ self.logger.debug("Acquired replication connection with pid %s", self.walconn.get_backend_pid())
+ self.walcur = self.walconn.cursor()
+
+ # clean up old slot
+ if self.slot_exists():
+ self.walcur.drop_replication_slot(SLOT_NAME)
+
+ # Create slot to use in testing
+ self.walcur.create_replication_slot(SLOT_NAME, output_plugin='pglogical_output')
+ slotinfo = self.walcur.fetchone()
+ self.logger.debug("Got slot info %s", slotinfo)
+
+
+ def cleanup(self):
+ self.logger.debug("Closing walsender connection")
+
+ if self.walcur is not None:
+ self.walcur.close()
+ if self.walconn is not None:
+ self.walconn.close()
+
+ self.replication_started = False
+
+ self.drop_slot_when_inactive()
+ BaseDecodingInterface.cleanup(self)
+ self.logger.debug("Closed walsender connection")
+
+ def get_changes(self, kwargs = {}):
+ params_dict = self._get_changes_params(kwargs)
+
+ if not self.replication_started:
+ self.walcur.start_replication(slot_name=SLOT_NAME,
+ options={k: v for k, v in params_dict.iteritems() if v is not None})
+ self.replication_started = True
+ try:
+ while True:
+ # There's never any "done" or "last message", so just keep
+ # reading as long as the caller asks. If select times out,
+ # a normal client would send feedback. We'll treat it as a
+ # failure instead, since the caller asked for a message we
+ # are apparently not going to receive.
+ message = self.walcur.read_replication_message(decode=False)
+ if message is None:
+ self.logger.debug("No message pending, select()ing with timeout %s", self.select_timeout)
+ sel = select.select([self.walcur], [], [], self.select_timeout)
+ if not sel[0]:
+ raise IOError("Server didn't send an expected message before timeout")
+ else:
+ if len(message.payload) < 200:
+ self.logger.debug("payload: %s", repr(message.payload))
+ else:
+ self.logger.debug("payload (truncated): %s...", repr(message.payload)[:200])
+ yield ReplicationMessage((message.data_start, None, message.payload))
+ except psycopg2.InternalError, ex:
+ self.logger.debug("While retrieving a message: sqlstate=%s", ex.pgcode, exc_info=True)
+
+
+
+
+class PGLogicalOutputTest(unittest.TestCase):
+
+ connstring = "dbname=postgres host=localhost"
+ interface = None
+
+ def setUp(self):
+ # A counter we can increment each time we reconnet with decoding,
+ # for logging purposes.
+ self.decoding_generation = 0
+
+ # Set up our logger
+ self.logger = logging.getLogger(self.__class__.__name__)
+ self.loghandler = logging.StreamHandler()
+ for handler in self.logger.handlers:
+ self.logger.removeHandler(handler)
+ self.logger.addHandler(self.loghandler)
+ self.logger.setLevel(os.environ.get('PGLOGICALTEST_LOGLEVEL', 'INFO'))
+
+ # Get connections for test classes to use to run SQL
+ self.conn = psycopg2.connect(self.connstring, connection_factory=psycopg2.extras.LoggingConnection)
+ self.conn.initialize(self.logger.getChild('sql'))
+ self.cur = self.conn.cursor()
+
+ if hasattr(self, 'set_up'):
+ self.set_up()
+
+ def tearDown(self):
+ if hasattr(self, 'tear_down'):
+ self.tear_down()
+
+ if self.conn is not None:
+ self.conn.close()
+
+ def doCleanups(self):
+ if self.interface:
+ self.interface.cleanup()
+
+ def reconnect_decoding(self):
+ """
+ Close the logical decoding connection and re-establish it.
+
+ This is useful when we want to restart decoding with different parameters,
+ since in walsender mode there's no way to end a decoding session once
+ begun.
+ """
+ if self.interface is not None:
+ self.logger.debug("Disconnecting old decoding session and forcing reconnect")
+ self.interface.cleanup()
+
+ self.connect_decoding()
+
+ def connect_decoding(self):
+ """
+ Make a slot and establish a decoding connection.
+
+ Prior to this changes are not recorded, which is useful for setup.
+ """
+ self.decoding_generation += 1
+ fmt = logging.Formatter('%%(name)-50s w=%s %%(message)s' % (self.decoding_generation,))
+ self.loghandler.setFormatter(fmt)
+
+ if os.environ.get("PGLOGICALTEST_USEWALSENDER", None):
+ self.interface = WalsenderDecodingInterface(self.connstring, parentlogger=self.logger)
+ else:
+ self.interface = SQLDecodingInterface(self.connstring, parentlogger=self.logger)
+
+
+ def get_changes(self, kwargs = {}):
+ """
+ Get a stream of messages as a generator that may be read from
+ to fetch a new message each call. Messages are instances of
+ class ReplicationMessage .
+
+ The generator has helper methods for decoding particular
+ types of message, for validation, etc.
+ """
+ if self.interface is None:
+ raise ValueError("No logical decoding connection. Call connect_decoding()")
+
+ msg_gen = self.interface.get_changes(kwargs)
+ return ProtocolReader(msg_gen, tester=self, parentlogger=self.logger)
diff --git a/contrib/pglogical_output/test/pglogical_proto.py b/contrib/pglogical_output/test/pglogical_proto.py
new file mode 100644
index 0000000..e8642e3
--- /dev/null
+++ b/contrib/pglogical_output/test/pglogical_proto.py
@@ -0,0 +1,240 @@
+from StringIO import StringIO
+import struct
+import datetime
+
+class UnchangedField(object):
+ """Opaque placeholder object for a TOASTed field that didn't change"""
+ pass
+
+def readcstr(f):
+ buf = bytearray()
+ while True:
+ b = f.read(1)
+ if b is None or len(b) == 0:
+ if len(buf) == 0:
+ return None
+ else:
+ raise ValueError("non-terminated string at EOF")
+ elif b is '\0':
+ return str(buf)
+ else:
+ buf.append(b)
+
+class ReplicationMessage(object):
+ def __new__(cls, msg):
+ msgtype = msg[2][0]
+ if msgtype == "S":
+ cls = StartupMessage
+ elif msgtype == "B":
+ cls = BeginMessage
+ elif msgtype == "C":
+ cls = CommitMessage
+ elif msgtype == "O":
+ cls = OriginMessage
+ elif msgtype == "R":
+ cls = RelationMessage
+ elif msgtype == "I":
+ cls = InsertMessage
+ elif msgtype == "U":
+ cls = UpdateMessage
+ elif msgtype == "D":
+ cls = DeleteMessage
+ else:
+ raise Exception("Unknown message type %s", msgtype)
+
+ return super(ReplicationMessage, cls).__new__(cls)
+
+ def __init__(self, row):
+ self.lsn = row[0]
+ self.xid = row[1]
+ self.msg = row[2]
+
+ @property
+ def message_type(self):
+ return self.msg[0]
+
+ @property
+ def message(self):
+ return None
+
+ def __repr__(self):
+ return repr(self.message)
+
+ def parse_tuple(self, msg):
+ assert msg.read(1) == "T"
+ numcols = struct.unpack("!H", msg.read(2))[0]
+
+ cols = []
+ for i in xrange(0, numcols):
+ typ = msg.read(1)
+ if typ == 'n':
+ cols.append(None)
+ elif typ == 'u':
+ cols.append(UnchangedField())
+ else:
+ assert typ in ('i','b','t') #typ should be 'i'nternal-binary, 'b'inary, 't'ext
+ datalen = struct.unpack("!I", msg.read(4))[0]
+ cols.append(msg.read(datalen))
+
+ return cols
+
+class ChangeMessage(ReplicationMessage):
+ pass
+
+class TransactionMessage(ReplicationMessage):
+ pass
+
+class StartupMessage(ReplicationMessage):
+ @property
+ def message(self):
+ res = {"type": "S"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'S'
+ res['startup_msg_version'] = struct.unpack("b", msg.read(1))[0]
+ # Now split the null-terminated k/v strings
+ # and store as a dict, since we don't care about order.
+ params = {}
+ while True:
+ k = readcstr(msg)
+ if k is None:
+ break;
+ v = readcstr(msg)
+ if (v is None):
+ raise ValueError("Value for key %s missing, read key as last entry" % k)
+ params[k] = v
+ res['params'] = params
+
+ return res
+
+class BeginMessage(TransactionMessage):
+ @property
+ def message(self):
+ res = {"type": "B"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'B'
+ msg.read(1) # flags
+
+ lsn, time, xid = struct.unpack("!QQI", msg.read(20))
+ res['final_lsn'] = lsn
+ res['timestamp'] = datetime.datetime.fromtimestamp(time)
+ res['xid'] = xid
+
+ return res
+
+class OriginMessage(ReplicationMessage):
+ @property
+ def message(self):
+ res = {"type": "O"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'O'
+ msg.read(1) # flags
+
+ origin_lsn, namelen = struct.unpack("!QB", msg.read(9))
+ res['origin_lsn'] = origin_lsn
+ res['origin_name'] = msg.read(namelen)
+
+ return res
+
+class RelationMessage(ReplicationMessage):
+ @property
+ def message(self):
+ res = {"type": "R"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'R'
+ msg.read(1) # flags
+
+ relid, namelen = struct.unpack("!IB", msg.read(5))
+ res['relid'] = relid
+ res['namespace'] = msg.read(namelen)
+ namelen = struct.unpack("B", msg.read(1))[0]
+ res['relation'] = msg.read(namelen)
+
+ assert msg.read(1) == "A" # attributes
+ numcols = struct.unpack("!H", msg.read(2))[0]
+
+ cols = []
+ for i in xrange(0, numcols):
+ assert msg.read(1) == "C" # column
+ msg.read(1) # flags
+ assert msg.read(1) == "N" # name
+
+ namelen = struct.unpack("!H", msg.read(2))[0]
+ cols.append(msg.read(namelen))
+
+ res["columns"] = cols
+
+ return res
+
+class CommitMessage(TransactionMessage):
+ @property
+ def message(self):
+ res = {"type": "C"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'C'
+ msg.read(1) # flags
+
+ commit_lsn, end_lsn, time = struct.unpack("!QQQ", msg.read(24))
+ res['commit_lsn'] = commit_lsn
+ res['end_lsn'] = end_lsn
+ res['timestamp'] = datetime.datetime.fromtimestamp(time)
+
+ return res
+
+class InsertMessage(ChangeMessage):
+ @property
+ def message(self):
+ res = {"type": "I"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'I'
+ msg.read(1) # flags
+
+ res["relid"] = struct.unpack("!I", msg.read(4))[0]
+
+ assert msg.read(1) == "N"
+ res["newtup"] = self.parse_tuple(msg)
+
+ return res
+
+class UpdateMessage(ChangeMessage):
+ @property
+ def message(self):
+ res = {"type": "U"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'I'
+ msg.read(1) # flags
+
+ res["relid"] = struct.unpack("!I", msg.read(4))[0]
+
+ tuptyp = msg.read(1)
+ if tuptyp == "K":
+ res["keytup"] = self.parse_tuple(msg)
+ tuptyp = msg.read(1)
+
+ tuptyp == "N"
+ res["newtup"] = self.parse_tuple(msg)
+
+ return res
+
+class DeleteMessage(ChangeMessage):
+ @property
+ def message(self):
+ res = {"type": "D"}
+
+ msg = StringIO(self.msg)
+ msg.read(1) # 'I'
+ msg.read(1) # flags
+
+ res["relid"] = struct.unpack("!I", msg.read(4))[0]
+
+ assert msg.read(1) == "K"
+ res["keytup"] = self.parse_tuple(msg)
+
+ return res
+
diff --git a/contrib/pglogical_output/test/pglogical_protoreader.py b/contrib/pglogical_output/test/pglogical_protoreader.py
new file mode 100644
index 0000000..5d570e0
--- /dev/null
+++ b/contrib/pglogical_output/test/pglogical_protoreader.py
@@ -0,0 +1,112 @@
+import collections
+import logging
+
+class ProtocolReader(collections.Iterator):
+ """
+ A protocol generator wrapper that can validate a message before returning
+ it and has helpers for reading different message types.
+
+ The underlying message generator is stored as the message_generator
+ member, but you shouldn't consume any messages from it directly, since
+ that'll break validation if enabled.
+ """
+
+ startup_params = None
+
+ def __init__(self, message_generator, validator=None, tester=None, parentlogger=logging.getLogger('base')):
+ """
+ Build a protocol reader to wrap the passed message_generator, which
+ must return a ReplicationMessage instance when next() is called.
+
+ A validator class may be provided. If supplied, it must have
+ a validate(...) method taking a ReplicationMessage instance as
+ an argument. It should throw exceptions if it sees things it
+ doesn't like.
+
+ A tester class may be provided. This class should be an instance
+ of unittest.TestCase. If provided, unittest assertions are used
+ to check message types, etc.
+ """
+ self.logger = parentlogger.getChild(self.__class__.__name__)
+ self.message_generator = message_generator
+ self.validator = validator
+ self.tester = tester
+
+ def next(self):
+ """Validating for generator"""
+ msg = self.message_generator.next()
+ if self.validator:
+ self.validator.validate(msg)
+ return msg
+
+ def expect_msg(self, msgtype):
+ """Read a message and check it's type char is as specified"""
+ m = self.next()
+ # this is ugly, better suggestions welcome:
+ try:
+ if self.tester:
+ self.tester.assertEqual(m.message_type, msgtype)
+ elif m.message_type <> msgtype:
+ raise ValueError("Expected message %s but got %s", msgtype, m.message_type)
+ except Exception, ex:
+ self.logger.debug("Expecting %s msg, got %s, unexpected message was: %s", msgtype, m.message_type, m)
+ raise
+ return m
+
+ def expect_startup(self):
+ """Get startup message and return the message and params objects"""
+ m = self.expect_msg('S')
+ # this is ugly, better suggestions welcome:
+ if self.tester:
+ self.tester.assertEquals(m.message['startup_msg_version'], 1)
+ elif m.message['startup_msg_version'] <> 1:
+ raise ValueError("Expected startup_msg_version 1, got %s", m.message['startup_msg_version'])
+ self.startup_params = m.message['params']
+ return (m, self.startup_params)
+
+ def expect_begin(self):
+ """Read a message and ensure it's a begin"""
+ return self.expect_msg('B')
+
+ def expect_row_meta(self):
+ """Read a message and ensure it's a rowmeta message"""
+ return self.expect_msg('R')
+
+ def expect_commit(self):
+ """Read a message and ensure it's a commit"""
+ return self.expect_msg('C')
+
+ def expect_insert(self):
+ """Read a message and ensure it's an insert"""
+ return self.expect_msg('I')
+
+ def expect_update(self):
+ """Read a message and ensure it's an update"""
+ return self.expect_msg('U')
+
+ def expect_delete(self):
+ """Read a message and ensure it's a delete"""
+ return self.expect_msg('D')
+
+ def expect_origin(self):
+ """
+ Read a message and ensure it's a replication origin message.
+ """
+ return self.expect_msg('O')
+
+ def maybe_expect_origin(self):
+ """
+ If the upstream sends replication origins, read one, otherwise
+ do nothing and return None.
+
+ If the upstream is 9.4 then it'll always send replication origin
+ messages. For other upstreams they're sent only if enabled.
+
+ Requires that the startup message was read with expect_startup(..)
+ """
+ if self.startup_params is None:
+ raise ValueError("Startup message was not read with expect_startup()")
+ if self.startup_params['forward_changeset_origins'] == 't':
+ return self.expect_origin()
+ else:
+ return None
diff --git a/contrib/pglogical_output/test/test_basic.py b/contrib/pglogical_output/test/test_basic.py
new file mode 100644
index 0000000..f4e86c6
--- /dev/null
+++ b/contrib/pglogical_output/test/test_basic.py
@@ -0,0 +1,89 @@
+import random
+import string
+import unittest
+from base import PGLogicalOutputTest
+
+class BasicTest(PGLogicalOutputTest):
+ def rand_string(self, length):
+ return ''.join([random.choice(string.ascii_letters + string.digits) for n in xrange(length)])
+
+ def setUp(self):
+ PGLogicalOutputTest.setUp(self)
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE IF EXISTS test_changes;")
+ cur.execute("CREATE TABLE test_changes (cola serial PRIMARY KEY, colb timestamptz default now(), colc text);")
+ self.conn.commit()
+ self.connect_decoding()
+
+ def tearDown(self):
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE test_changes;")
+ self.conn.commit()
+ PGLogicalOutputTest.tearDown(self)
+
+ def test_changes(self):
+ cur = self.conn.cursor()
+ cur.execute("INSERT INTO test_changes(colb, colc) VALUES(%s, %s)", ('2015-08-08', 'foobar'))
+ cur.execute("INSERT INTO test_changes(colb, colc) VALUES(%s, %s)", ('2015-08-08', 'bazbar'))
+ self.conn.commit()
+
+ cur.execute("DELETE FROM test_changes WHERE cola = 1")
+ cur.execute("UPDATE test_changes SET colc = 'foobar' WHERE cola = 2")
+ self.conn.commit()
+
+ messages = self.get_changes()
+
+ # Startup msg
+ (m, params) = messages.expect_startup()
+
+ self.assertEquals(params['max_proto_version'], '1')
+ self.assertEquals(params['min_proto_version'], '1')
+
+ if int(params['pg_version_num'])/100 == 904:
+ self.assertEquals(params['forward_changeset_origins'], 'f')
+ self.assertEquals(params['forward_changesets'], 't')
+ else:
+ self.assertEquals(params['forward_changeset_origins'], 'f')
+ self.assertEquals(params['forward_changesets'], 'f')
+
+ anybool = ['t', 'f']
+ self.assertIn(params['binary.bigendian'], anybool)
+ self.assertIn(params['binary.internal_basetypes'], anybool)
+ self.assertIn(params['binary.binary_basetypes'], anybool)
+ self.assertIn(params['binary.float4_byval'], anybool)
+ self.assertIn(params['binary.float8_byval'], anybool)
+ self.assertIn(params['binary.integer_datetimes'], anybool)
+ self.assertIn(params['binary.maxalign'], ['4', '8'])
+ self.assertIn(params['binary.sizeof_int'], ['4', '8'])
+ self.assertIn(params['binary.sizeof_long'], ['4', '8'])
+
+ self.assertIn("encoding", params)
+ self.assertEquals(params['coltypes'], 'f')
+
+ self.assertIn('pg_catversion', params)
+ self.assertIn('pg_version', params)
+ self.assertIn('pg_version_num', params)
+
+ # two inserts in one tx
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'foobar\0')
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'bazbar\0')
+ messages.expect_commit()
+
+ # delete and update in one tx
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_delete()
+ self.assertEqual(m.message['keytup'][0], '1\0')
+ messages.expect_row_meta()
+ m = messages.expect_update()
+ self.assertEqual(m.message['newtup'][0], '2\0')
+ self.assertEqual(m.message['newtup'][2], 'foobar\0')
+ messages.expect_commit()
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/contrib/pglogical_output/test/test_binary_mode.py b/contrib/pglogical_output/test/test_binary_mode.py
new file mode 100644
index 0000000..da3b11f
--- /dev/null
+++ b/contrib/pglogical_output/test/test_binary_mode.py
@@ -0,0 +1,172 @@
+import random
+import string
+import unittest
+from base import PGLogicalOutputTest
+
+class BinaryModeTest(PGLogicalOutputTest):
+
+ def setUp(self):
+ PGLogicalOutputTest.setUp(self)
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE IF EXISTS test_binary;")
+ cur.execute("CREATE TABLE test_binary (colv bytea, colts timestamp);")
+ self.conn.commit()
+ self.connect_decoding()
+
+ def tearDown(self):
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE IF EXISTS test_binary;")
+ self.conn.commit()
+ PGLogicalOutputTest.tearDown(self)
+
+ def probe_for_server_params(self):
+ cur = self.conn.cursor()
+
+ # Execute a dummy transaction so we have something to decode
+ cur.execute("INSERT INTO test_binary values (decode('ff','hex'), NULL);")
+ self.conn.commit()
+
+ # Make a connection to decode the dummy tx. We're just doing this
+ # so we can capture the startup response from the server, then
+ # we'll disconnect and reconnect with the binary settings captured
+ # from the server to ensure we make a request that'll get binary
+ # mode enabled.
+ messages = self.get_changes()
+
+ (m, params) = messages.expect_startup()
+
+ # Check we got everything we expected from the startup params
+ expected_params = ['pg_version_num', 'binary.bigendian',
+ 'binary.sizeof_datum', 'binary.sizeof_int',
+ 'binary.sizeof_long', 'binary.float4_byval',
+ 'binary.float8_byval', 'binary.integer_datetimes',
+ 'binary.internal_basetypes', 'binary.binary_basetypes',
+ 'binary.basetypes_major_version']
+ for pn in expected_params:
+ self.assertTrue(pn in params, msg="Expected startup msg param binary.basetypes_major_version absent")
+
+ self.assertEquals(int(params['pg_version_num'])/100,
+ int(params['binary.basetypes_major_version']),
+ msg="pg_version_num/100 <> binary.basetypes_major_version")
+
+ # We didn't ask for it, so binary and send/recv must be disabled
+ self.assertEquals(params['binary.internal_basetypes'], 'f')
+ self.assertEquals(params['binary.binary_basetypes'], 'f')
+
+ # Read and discard the fields of our dummy tx
+ messages.expect_begin()
+ messages.expect_row_meta()
+ messages.expect_insert()
+ messages.expect_commit()
+
+ cur.close()
+
+ # We have to disconnect and reconnect if using walsender so that
+ # we can start a new replication session
+ self.logger.debug("before: Interface is %s", self.interface)
+ self.reconnect_decoding()
+ self.logger.debug("after: Interface is %s", self.interface)
+
+ return params
+
+ def test_binary_mode(self):
+ params = self.probe_for_server_params()
+ major_version = int(params['pg_version_num'])/100
+
+ cur = self.conn.cursor()
+
+ # Now that we know the server's parameters, do another transaction and
+ # decode it in binary mode using the format we know the server speaks.
+ cur.execute("INSERT INTO test_binary values (decode('aa','hex'), TIMESTAMP '2000-01-02 12:34:56');")
+ self.conn.commit()
+
+ messages = self.get_changes({
+ 'binary.want_internal_basetypes': 't',
+ 'binary.want_binary_basetypes': 't',
+ 'binary.basetypes_major_version': str(major_version),
+ 'binary.bigendian' : params['binary.bigendian'],
+ 'binary.sizeof_datum' : params['binary.sizeof_datum'],
+ 'binary.sizeof_int' : params['binary.sizeof_int'],
+ 'binary.sizeof_long' : params['binary.sizeof_long'],
+ 'binary.float4_byval' : params['binary.float4_byval'],
+ 'binary.float8_byval' : params['binary.float8_byval'],
+ 'binary.integer_datetimes' : params['binary.integer_datetimes']
+ })
+
+ # Decode the startup message
+ (m, params) = messages.expect_startup()
+ # Binary mode should be enabled since we sent the params the server wants
+ self.assertEquals(params['binary.internal_basetypes'], 't')
+ # send/recv mode is implied by binary mode
+ self.assertEquals(params['binary.binary_basetypes'], 't')
+
+ # Decode the transaction we sent
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+
+ # and verify that the message fields are in the expected binary representation
+ self.assertEqual(m.message['newtup'][0], '\x05\xaa')
+ # FIXME this is probably wrong on bigendian
+ self.assertEqual(m.message['newtup'][1], '\x00\x7c\xb1\xa9\x1e\x00\x00\x00')
+
+ messages.expect_commit()
+
+ def test_sendrecv_mode(self):
+ params = self.probe_for_server_params()
+ major_version = int(params['pg_version_num'])/100
+
+ cur = self.conn.cursor()
+
+ # Now that we know the server's parameters, do another transaction and
+ # decode it in binary mode using the format we know the server speaks.
+ cur.execute("INSERT INTO test_binary values (decode('aa','hex'), TIMESTAMP '2000-01-02 12:34:56');")
+ self.conn.commit()
+
+ # Send options that don't match the server's binary mode, so it falls
+ # back to send/recv even though we requested binary too.
+ if int(params['binary.sizeof_long']) == 8:
+ want_sizeof_long = 4
+ elif int(params['binary.sizeof_long']) == 4:
+ want_sizeof_long = 8
+ else:
+ self.fail("What platform has sizeof(long) == %s anyway?" % params['binary.sizeof_long'])
+
+ messages = self.get_changes({
+ # Request binary even though we know we won't get it
+ 'binary.want_internal_basetypes': 't',
+ # and expect to fall back to send/recv
+ 'binary.want_binary_basetypes': 't',
+ 'binary.basetypes_major_version': str(major_version),
+ 'binary.bigendian' : params['binary.bigendian'],
+ 'binary.sizeof_datum' : params['binary.sizeof_datum'],
+ 'binary.sizeof_int' : params['binary.sizeof_int'],
+ 'binary.sizeof_long' : want_sizeof_long,
+ 'binary.float4_byval' : params['binary.float4_byval'],
+ 'binary.float8_byval' : params['binary.float8_byval'],
+ 'binary.integer_datetimes' : params['binary.integer_datetimes']
+ })
+
+ # Decode the startup message
+ (m, params) = messages.expect_startup()
+ # Binary mode should be disabled because we aren't compatible
+ self.assertEquals(params['binary.internal_basetypes'], 'f')
+ # send/recv mode should be on, since we're compatible with the same
+ # major version.
+ self.assertEquals(params['binary.binary_basetypes'], 't')
+
+ # Decode the transaction we sent
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+
+ # and verify that the message fields are in the expected send/recv representation
+ # The text field lacks the length prefix
+ self.assertEqual(m.message['newtup'][0], '\xaa')
+ # and the timestamp is in network byte order
+ self.assertEqual(m.message['newtup'][1], '\x00\x00\x00\x1e\xa9\xb1\x7c\x00')
+
+ messages.expect_commit()
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/contrib/pglogical_output/test/test_filter.py b/contrib/pglogical_output/test/test_filter.py
new file mode 100644
index 0000000..fce827b
--- /dev/null
+++ b/contrib/pglogical_output/test/test_filter.py
@@ -0,0 +1,182 @@
+import random
+import string
+import unittest
+import pprint
+from base import PGLogicalOutputTest
+
+class FilterTest(PGLogicalOutputTest):
+ def rand_string(self, length):
+ return ''.join([random.choice(string.ascii_letters + string.digits) for n in xrange(length)])
+
+ def set_up(self):
+ cur = self.conn.cursor()
+ cur.execute("DROP EXTENSION IF EXISTS pglogical_output_plhooks;")
+ cur.execute("DROP TABLE IF EXISTS test_changes, test_changes_filter;")
+ cur.execute("DROP FUNCTION IF EXISTS test_filter(regclass, \"char\", text)")
+ cur.execute("DROP FUNCTION IF EXISTS test_action_filter(regclass, \"char\", text)")
+ cur.execute("CREATE TABLE test_changes (cola serial PRIMARY KEY, colb timestamptz default now(), colc text);")
+ cur.execute("CREATE TABLE test_changes_filter (cola serial PRIMARY KEY, colb timestamptz default now(), colc text);")
+ cur.execute("CREATE EXTENSION pglogical_output_plhooks;")
+
+
+ # Filter function that filters out (removes) all changes
+ # in tables named *_filter*
+ cur.execute("""
+ CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+ returns bool stable language plpgsql AS $$
+ BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+ END
+ $$;
+ """)
+
+ # function to filter out Deletes and Updates - Only Inserts pass through
+ cur.execute("""
+ CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+ returns bool stable language plpgsql AS $$
+ BEGIN
+ RETURN action NOT IN ('U', 'D');
+ END
+ $$;
+ """)
+
+ self.conn.commit()
+ self.connect_decoding()
+
+ def tear_down(self):
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE test_changes, test_changes_filter;")
+ cur.execute("DROP FUNCTION test_filter(regclass, \"char\", text)");
+ cur.execute("DROP FUNCTION test_action_filter(regclass, \"char\", text)");
+ cur.execute("DROP EXTENSION pglogical_output_plhooks;")
+ self.conn.commit()
+
+ def exec_changes(self):
+ """Execute a stream of changes we can process via various filters"""
+ cur = self.conn.cursor()
+ cur.execute("INSERT INTO test_changes(colb, colc) VALUES(%s, %s)", ('2015-08-08', 'foobar'))
+ cur.execute("INSERT INTO test_changes_filter(colb, colc) VALUES(%s, %s)", ('2015-08-08', 'foobar'))
+ cur.execute("INSERT INTO test_changes(colb, colc) VALUES(%s, %s)", ('2015-08-08', 'bazbar'))
+ self.conn.commit()
+
+ cur.execute("INSERT INTO test_changes_filter(colb, colc) VALUES(%s, %s)", ('2015-08-08', 'bazbar'))
+ self.conn.commit()
+
+ cur.execute("UPDATE test_changes set colc = 'oobar' where cola=1")
+ cur.execute("UPDATE test_changes_filter set colc = 'oobar' where cola=1")
+ self.conn.commit()
+
+ cur.execute("DELETE FROM test_changes where cola=2")
+ cur.execute("DELETE FROM test_changes_filter where cola=2")
+ self.conn.commit()
+
+
+ def test_filter(self):
+ self.exec_changes();
+
+ params = {
+ 'hooks.setup_function': 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook': 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg': 'foo'
+ }
+
+ messages = self.get_changes(params)
+
+ (m, params) = messages.expect_startup()
+
+ self.assertIn('hooks.row_filter_enabled', m.message['params'])
+ self.assertEquals(m.message['params']['hooks.row_filter_enabled'], 't')
+
+ # two inserts into test_changes, the test_changes_filter insert is filtered out
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'foobar\0')
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'bazbar\0')
+ messages.expect_commit()
+
+ # just an empty tx as the test_changes_filter insert is filtered out
+ messages.expect_begin()
+ messages.expect_commit()
+
+ # 1 update each into test_changes and test_changes_filter
+ # update of test_changes_filter is filtered out
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_update()
+ self.assertEqual(m.message['newtup'][0], '1\0')
+ self.assertEqual(m.message['newtup'][2], 'oobar\0')
+ messages.expect_commit()
+
+ # 1 delete each into test_changes and test_changes_filter
+ # delete of test_changes_filter is filtered out
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_delete()
+ self.assertEqual(m.message['keytup'][0], '2\0')
+ messages.expect_commit()
+
+ def test_action_filter(self):
+ self.exec_changes();
+
+ params = {
+ 'hooks.setup_function': 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook': 'public.test_action_filter'
+ }
+
+ messages = self.get_changes(params)
+
+ (m, params) = messages.expect_startup()
+
+ self.assertIn('hooks.row_filter_enabled', params)
+ self.assertEquals(params['hooks.row_filter_enabled'], 't')
+
+ # two inserts into test_changes, the test_changes_filter insert is filtered out
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'foobar\0')
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'foobar\0')
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'bazbar\0')
+ messages.expect_commit()
+
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][2], 'bazbar\0')
+ messages.expect_commit()
+
+ # just empty tx as updates are filtered out
+ messages.expect_begin()
+ messages.expect_commit()
+
+ # just empty tx as deletes are filtered out
+ messages.expect_begin()
+ messages.expect_commit()
+
+# def test_hooks(self):
+# params = {
+# 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+# 'pglo_plhooks.startup_hook', 'pglo_plhooks_demo_startup',
+# 'pglo_plhooks.row_filter_hook', 'pglo_plhooks_demo_row_filter',
+# 'pglo_plhooks.txn_filter_hook', 'pglo_plhooks_demo_txn_filter',
+# 'pglo_plhooks.shutdown_hook', 'pglo_plhooks_demo_shutdown',
+# 'pglo_plhooks.client_hook_arg', 'test data'
+# }
+#
+ def test_validation(self):
+ with self.assertRaises(Exception):
+ self.get_changes({'hooks.row_filter': 'public.foobar'}).next()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/contrib/pglogical_output/test/test_parameters.py b/contrib/pglogical_output/test/test_parameters.py
new file mode 100644
index 0000000..9cfe4e1
--- /dev/null
+++ b/contrib/pglogical_output/test/test_parameters.py
@@ -0,0 +1,80 @@
+import random
+import string
+import unittest
+from base import PGLogicalOutputTest
+import psycopg2
+
+class ParametersTest(PGLogicalOutputTest):
+
+ def setUp(self):
+ PGLogicalOutputTest.setUp(self)
+ self.cur.execute("DROP TABLE IF EXISTS blah;")
+ self.cur.execute("CREATE TABLE blah(id integer);")
+ self.conn.commit()
+ self.connect_decoding()
+
+ def tearDown(self):
+ self.cur.execute("DROP TABLE blah;")
+ self.conn.commit()
+ PGLogicalOutputTest.tearDown(self)
+
+ def test_protoversion(self):
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'startup_params_format': 'borkbork'}))
+
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'startup_params_format': '2'}))
+
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'startup_params_format': None}))
+
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'max_proto_version': None}))
+
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'min_proto_version': None}))
+
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'min_proto_version': '2'}))
+
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'max_proto_version': '0'}))
+
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'max_proto_version': 'borkbork'}))
+
+ def test_unknown_params(self):
+ # Should get ignored
+ self.do_dummy_tx()
+ self.get_startup_msg(self.get_changes({'unknown_parameter': 'unknown'}))
+
+ def test_unknown_params(self):
+ # Should get ignored
+ self.do_dummy_tx()
+ self.get_startup_msg(self.get_changes({'unknown.some_param': 'unknown'}))
+
+ def test_encoding_missing(self):
+ # Should be ignored, server should send reply params
+ messages = self.get_changes({'expected_encoding': None})
+
+ def test_encoding_bogus(self):
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'expected_encoding': 'gobblegobble'}))
+
+ def test_encoding_differs(self):
+ with self.assertRaises(psycopg2.DatabaseError):
+ list(self.get_changes({'expected_encoding': 'LATIN-1'}))
+
+ def do_dummy_tx(self):
+ """force a dummy tx so there's something to decode"""
+ self.cur.execute("INSERT INTO blah(id) VALUES (1)")
+ self.conn.commit()
+
+ def get_startup_msg(self, messages):
+ """Read and return the startup message"""
+ m = messages.next()
+ self.assertEqual(m.message_type, 'S')
+ return m
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/contrib/pglogical_output/test/test_replication_origin.py b/contrib/pglogical_output/test/test_replication_origin.py
new file mode 100644
index 0000000..d65b5d0
--- /dev/null
+++ b/contrib/pglogical_output/test/test_replication_origin.py
@@ -0,0 +1,327 @@
+import random
+import string
+import unittest
+from base import PGLogicalOutputTest
+
+class ReplicationOriginTest(PGLogicalOutputTest):
+ """
+ Tests for handling of changeset forwarding and replication origins.
+
+ These tests have to deal with a bit of a wrinkle: if the decoding plugin
+ is running in PostgreSQL 9.4 the lack of replication origins support means
+ that we cannot send replication origin information, and we always forward
+ transactions from other peer nodes. So it produces output you can't get
+ from 9.5+: all changesets forwarded, but without origin information.
+
+ 9.4 also lacks the functions for setting up replication origins, so we
+ have to special-case that.
+ """
+
+ fake_upstream_origin_name = "pglogical_test_fake_upstream";
+ fake_xact_lsn = '14/abcdef0'
+ fake_xact_timestamp = '2015-10-08 12:13:14.1234'
+
+ def setUp(self):
+ PGLogicalOutputTest.setUp(self)
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE IF EXISTS test_origin;")
+ if self.conn.server_version/100 != 904:
+ cur.execute("""
+ SELECT pg_replication_origin_drop(%s)
+ FROM pg_replication_origin
+ WHERE roname = %s;
+ """,
+ (self.fake_upstream_origin_name, self.fake_upstream_origin_name))
+ self.conn.commit()
+
+ cur.execute("CREATE TABLE test_origin (cola serial PRIMARY KEY, colb timestamptz default now(), colc text);")
+ self.conn.commit()
+
+ self.is95plus = self.conn.server_version/100 > 904
+
+ if self.is95plus:
+ # Create the replication origin for the fake remote node
+ cur.execute("SELECT pg_replication_origin_create(%s);", (self.fake_upstream_origin_name,))
+ self.conn.commit()
+
+ # Ensure that commit timestamps are enabled.
+ cur.execute("SHOW track_commit_timestamp");
+ if cur.fetchone()[0] != 'on':
+ raise ValueError("This test requires track_commit_timestamp to be on")
+ self.conn.commit()
+
+ self.connect_decoding()
+
+ def tearDown(self):
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE IF EXISTS test_origin;")
+ self.conn.commit()
+ self.teardown_replication_session_origin(cur);
+ self.conn.commit()
+ PGLogicalOutputTest.tearDown(self)
+
+ def setup_replication_session_origin(self, cur):
+ """Sets session-level replication origin info. Ignored on 9.4."""
+ if self.conn.get_transaction_status() != 0:
+ raise ValueError("Transaction open or aborted, expected no open transaction")
+
+ if self.is95plus:
+ # Set our session up so it appears to be replaying from the nonexistent remote node
+ cur.execute("SELECT pg_replication_origin_session_setup(%s);", (self.fake_upstream_origin_name,))
+ self.conn.commit()
+
+ def setup_xact_origin(self, cur, origin_lsn, origin_commit_timestamp):
+ """Sets transaction-level replication origin info. Ignored on 9.4. Implicitly begins a tx."""
+ if self.conn.get_transaction_status() != 0:
+ raise ValueError("Transaction open or aborted, expected no open transaction")
+
+ if self.is95plus:
+ # Run transactions that seem to come from the remote node
+ cur.execute("SELECT pg_replication_origin_xact_setup(%s, %s);", (origin_lsn, origin_commit_timestamp))
+
+ def reset_replication_session_origin(self, cur):
+ """
+ Reset session's replication origin setup to defaults.
+
+ Always executes an empty transaction on 9.5+; does nothing
+ on 9.4.
+ """
+ if self.conn.get_transaction_status() != 0:
+ raise ValueError("Transaction open or aborted, expected no open transaction")
+ if self.is95plus:
+ cur.execute("SELECT pg_replication_origin_session_reset();")
+ self.conn.commit()
+
+ def teardown_replication_session_origin(self, cur):
+ if self.conn.get_transaction_status() != 0:
+ raise ValueError("Transaction open or aborted, expected no open transaction")
+ if self.is95plus:
+ cur.execute("SELECT pg_replication_origin_session_is_setup()")
+ if cur.fetchone()[0] == 't':
+ cur.execute("SELECT pg_replication_origin_session_reset();")
+ self.conn.commit()
+ cur.execute("SELECT pg_replication_origin_drop(%s);", (self.fake_upstream_origin_name,))
+ self.conn.commit()
+
+ def expect_origin_progress(self, cur, lsn):
+ if self.is95plus:
+ initialtxstate = self.conn.get_transaction_status()
+ if initialtxstate not in (0,2):
+ raise ValueError("Expected open valid tx or no tx")
+ cur.execute("SELECT local_id, external_id, remote_lsn FROM pg_show_replication_origin_status()")
+ if lsn is not None:
+ (local_id, external_id, remote_lsn) = cur.fetchone()
+ self.assertEquals(local_id, 1)
+ self.assertEquals(external_id, self.fake_upstream_origin_name)
+ self.assertEquals(remote_lsn.lower(), lsn.lower())
+ self.assertIsNone(cur.fetchone(), msg="Expected only one replication origin to exist")
+ if initialtxstate == 0:
+ self.conn.commit()
+
+ def run_test_transactions(self, cur):
+ """
+ Run a set of transactions with and without a replication origin set.
+
+ This simulates a mix of local transactions and remotely-originated
+ transactions being applied by a pglogical downstream or some other
+ replication-origin aware replication agent.
+
+ On 9.5+ the with-origin transaction simulates what the apply side of a
+ logical replication downstream would do by setting replication origin
+ information on the session and transaction. So to the server it's just
+ like this transaction was forwarded from another node.
+
+ On 9.4 it runs like a locally originated tx because 9.4 lacks origins
+ support.
+
+ All the tests will decode the same series of transactions, but with
+ different connection settings.
+ """
+
+ self.expect_origin_progress(cur, None)
+
+ # Some locally originated tx's for data we'll then modify
+ cur.execute("INSERT INTO test_origin(colb, colc) VALUES(%s, %s)", ('2015-10-08', 'foobar'))
+ self.assertEquals(cur.rowcount, 1)
+ cur.execute("INSERT INTO test_origin(colb, colc) VALUES(%s, %s)", ('2015-10-08', 'bazbar'))
+ self.assertEquals(cur.rowcount, 1)
+ self.conn.commit()
+
+ self.expect_origin_progress(cur, None)
+
+ # Now the fake remotely-originated tx
+ self.setup_replication_session_origin(cur)
+ self.setup_xact_origin(cur, self.fake_xact_lsn, self.fake_xact_timestamp)
+ # Some remotely originated inserts
+ cur.execute("INSERT INTO test_origin(colb, colc) VALUES(%s, %s)", ('2016-10-08', 'fakeor'))
+ self.assertEquals(cur.rowcount, 1)
+ cur.execute("INSERT INTO test_origin(colb, colc) VALUES(%s, %s)", ('2016-10-08', 'igin'))
+ self.assertEquals(cur.rowcount, 1)
+ # Delete a tuple that was inserted locally
+ cur.execute("DELETE FROM test_origin WHERE colb = '2015-10-08' and colc = 'foobar'")
+ self.assertEquals(cur.rowcount, 1)
+ # modify a tuple that was inserted locally
+ cur.execute("UPDATE test_origin SET colb = '2016-10-08' where colc = 'bazbar'")
+ self.assertEquals(cur.rowcount, 1)
+ self.conn.commit()
+
+ self.expect_origin_progress(cur, self.fake_xact_lsn)
+
+ # Reset replication origin to return to locally originated tx's
+ self.reset_replication_session_origin(cur)
+
+ self.expect_origin_progress(cur, self.fake_xact_lsn)
+
+ # and finally use a local tx to modify remotely originated transactions
+ # Delete and modify remotely originated tuples
+ cur.execute("DELETE FROM test_origin WHERE colc = 'fakeor'")
+ self.assertEquals(cur.rowcount, 1)
+ cur.execute("UPDATE test_origin SET colb = '2015-10-08' WHERE colc = 'igin'")
+ self.assertEquals(cur.rowcount, 1)
+ # and insert a new row mainly to verify that the origin reset was respected
+ cur.execute("INSERT INTO test_origin(colb, colc) VALUES (%s, %s)", ('2017-10-08', 'blahblah'))
+ self.assertEquals(cur.rowcount, 1)
+ self.conn.commit()
+
+ self.expect_origin_progress(cur, self.fake_xact_lsn)
+
+ def decode_test_transactions(self, messages, expect_origins, expect_forwarding):
+ """
+ Decode the transactions from run_test_transactions, varying the
+ expected output based on whether we've been told we should be getting
+ origin messages, and whether we should be getting forwarded transaction
+ data.
+
+ This intentionally doesn't use maybe_expect_origin() to make sure it's
+ testing what the unit test specifies, not what the server sent in the
+ startup message.
+ """
+ # two inserts in one locally originated tx. No forwarding. Local tx's
+ # never get origins.
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ messages.expect_commit()
+
+ # The remotely originated transaction is still replayed when forwarding
+ # is off, but on 9.5+ the data from it is omitted.
+ #
+ # An origin message will be received only if on 9.5+.
+ if expect_forwarding:
+ messages.expect_begin()
+ if expect_origins:
+ messages.expect_origin()
+ # 9.4 forwards unconditionally
+ m = messages.expect_row_meta()
+ m = messages.expect_insert()
+ m = messages.expect_row_meta()
+ m = messages.expect_insert()
+ m = messages.expect_row_meta()
+ m = messages.expect_delete()
+ m = messages.expect_row_meta()
+ m = messages.expect_update()
+ messages.expect_commit()
+
+ # The second locally originated tx modifies the remotely
+ # originated tuples. It's locally originated so no origin
+ # message is sent.
+ messages.expect_begin()
+ m = messages.expect_row_meta()
+ m = messages.expect_delete()
+ m = messages.expect_row_meta()
+ m = messages.expect_update()
+ m = messages.expect_row_meta()
+ m = messages.expect_insert()
+ messages.expect_commit()
+
+ def test_forwarding_not_requested_95plus(self):
+ """
+ For this test case we don't ask for forwarding to be enabled.
+
+ For 9.5+ we should get only transactions originated locally, so any transaction
+ with an origin set will be ignored. No origin info messages will be sent.
+ """
+ cur = self.conn.cursor()
+
+ if not self.is95plus:
+ self.skipTest("Cannot run forwarding-off origins-off test on a PostgreSQL 9.4 server")
+
+ self.run_test_transactions(cur)
+
+ # Forwarding not requested.
+ messages = self.get_changes({'forward_changesets':'f'})
+
+ # Startup msg
+ (m, params) = messages.expect_startup()
+
+ # We should not get origins, ever
+ self.assertEquals(params['forward_changeset_origins'], 'f')
+ # Changeset forwarding off respected by 9.5+
+ self.assertEquals(params['forward_changesets'], 'f')
+
+ # decode, expecting no origins
+ self.decode_test_transactions(messages, False, False)
+
+ # Upstream doesn't send origin correctly yet
+ @unittest.skip("Doesn't work yet")
+ def test_forwarding_requested_95plus(self):
+ """
+ In this test we request that forwarding be enabled. We'll get
+ forwarded transactions and origin messages for them.
+ """
+ cur = self.conn.cursor()
+
+ if not self.is95plus:
+ self.skipTest("Cannot run forwarding-on with-origins test on a PostgreSQL 9.4 server")
+
+ self.run_test_transactions(cur)
+
+ #client requests to forward changesets
+ messages = self.get_changes({'forward_changesets': 't'})
+
+ # Startup msg
+ (m, params) = messages.expect_startup()
+
+ # Changeset forwarding is forced on by 9.4 and was requested
+ # for 9.5+ so should always be on.
+ self.assertEquals(params['forward_changesets'], 't')
+ # 9.5+ will always forward origins if cset forwarding is
+ # requested.
+ self.assertEquals(params['forward_changeset_origins'], 't')
+
+ # Decode, expecting forwarding, and expecting origins unless 9.4
+ self.decode_test_transactions(messages, self.is95plus, True)
+
+ def test_forwarding_not_requested_94(self):
+ """
+ For this test case we don't ask for forwarding to be enabled.
+
+ For 9.4, we should get all transactions, even those that were originated "remotely".
+ 9.4 doesn't support replication identifiers so we couldn't tell the server that the
+ tx's were applied from a remote node, and it'd have to way to store that info anyway.
+ """
+ cur = self.conn.cursor()
+
+ if self.is95plus:
+ self.skipTest("9.4-specific test doesn't make sense on this server version")
+
+ self.run_test_transactions(cur)
+
+ # Forwarding not requested.
+ messages = self.get_changes({'forward_changesets':'f'})
+
+ # Startup msg
+ (m, params) = messages.expect_startup()
+
+ # We should not get origins, ever
+ self.assertEquals(params['forward_changeset_origins'], 'f')
+ # Changeset forwarding is forced on by 9.4
+ self.assertEquals(params['forward_changesets'], 't')
+
+ # decode, expecting no origins, and forwarding
+ self.decode_test_transactions(messages, False, True)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/contrib/pglogical_output/test/test_tuple_fields.py b/contrib/pglogical_output/test/test_tuple_fields.py
new file mode 100644
index 0000000..eb37529
--- /dev/null
+++ b/contrib/pglogical_output/test/test_tuple_fields.py
@@ -0,0 +1,159 @@
+import random
+import string
+import unittest
+from pglogical_proto import UnchangedField
+from base import PGLogicalOutputTest
+
+class TupleFieldsTest(PGLogicalOutputTest):
+
+ def setUp(self):
+ PGLogicalOutputTest.setUp(self)
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE IF EXISTS test_tuplen;")
+ #teardown after test
+ cur.execute("DROP TABLE IF EXISTS test_text;")
+ cur.execute("DROP TABLE IF EXISTS test_binary;")
+ cur.execute("DROP TABLE IF EXISTS toasttest;")
+
+ cur.execute("CREATE TABLE test_tuplen (cola serial PRIMARY KEY, colb timestamptz default now(), colc text);")
+ cur.execute("CREATE TABLE toasttest(descr text, cnt int DEFAULT 0, f1 text, f2 text);")
+ cur.execute("CREATE TABLE test_text (cola text, colb text);")
+ cur.execute("CREATE TABLE test_binary (colv bytea);")
+
+ self.conn.commit()
+ self.connect_decoding()
+
+ def tearDown(self):
+ cur = self.conn.cursor()
+ cur.execute("DROP TABLE IF EXISTS test_tuplen;")
+ #teardown after test
+ cur.execute("DROP TABLE IF EXISTS test_text;")
+ cur.execute("DROP TABLE IF EXISTS test_binary;")
+ cur.execute("DROP TABLE toasttest;")
+ self.conn.commit()
+
+ PGLogicalOutputTest.tearDown(self)
+
+ def test_null_tuple_field(self):
+ """Make sure null in tuple fields is sent as a 'n' row-value message"""
+ cur = self.conn.cursor()
+
+ cur.execute("INSERT INTO test_tuplen(colb, colc) VALUES('2015-08-08', null)")
+ # testing 'n'ull fields
+ self.conn.commit()
+
+ messages = self.get_changes()
+
+ messages.expect_startup()
+
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ # 'n'ull is reported as None by the test tuple reader
+ self.assertEqual(m.message['newtup'][2], None)
+ messages.expect_commit()
+
+ def test_unchanged_toasted_tuple_field(self):
+ """
+ Large TOASTed fields are sent as 'u'nchanged if they're from an UPDATE
+ and the UPDATE didn't change the TOASTed field, just other fields in the
+ same tuple.
+ """
+
+ # TODO: A future version should let us force unpacking of TOASTed fields,
+ # see bug #19
+
+ cur = self.conn.cursor()
+
+ cur.execute("INSERT INTO toasttest(descr, f1, f2) VALUES('one-toasted', repeat('1234567890',30000), 'atext');")
+ self.conn.commit()
+
+ # testing 'u'nchanged tuples
+ cur.execute("UPDATE toasttest SET cnt = 2 WHERE descr = 'one-toasted'")
+ self.conn.commit()
+
+ # but make sure they're replicated when actually changed
+ cur.execute("UPDATE toasttest SET cnt = 3, f1 = repeat('0987654321',25000) WHERE descr = 'one-toasted';")
+ self.conn.commit()
+
+
+ messages = self.get_changes()
+
+ # Startup msg
+ messages.expect_startup()
+
+ # consume the insert
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][0], 'one-toasted\0')
+ self.assertEqual(m.message['newtup'][1], '0\0') # default of cnt field
+ self.assertEqual(m.message['newtup'][2], '1234567890'*30000 + '\0')
+ self.assertEqual(m.message['newtup'][3], 'atext\0')
+ messages.expect_commit()
+
+
+ # First UPDATE
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_update()
+ self.assertEqual(m.message['newtup'][0], 'one-toasted\0')
+ self.assertEqual(m.message['newtup'][1], '2\0')
+ # The big value is TOASTed, and since we didn't change it, it's sent
+ # as an unchanged field marker.
+ self.assertIsInstance(m.message['newtup'][2], UnchangedField)
+ # While unchanged, 'atext' is small enough that it's not stored out of line
+ # so it's written despite being unchanged.
+ self.assertEqual(m.message['newtup'][3], 'atext\0')
+ messages.expect_commit()
+
+
+
+ # Second UPDATE
+ messages.expect_begin()
+ messages.expect_row_meta()
+ m = messages.expect_update()
+ self.assertEqual(m.message['newtup'][0], 'one-toasted\0')
+ self.assertEqual(m.message['newtup'][1], '3\0')
+ # this time we changed the TOASTed field, so it's been sent
+ self.assertEqual(m.message['newtup'][2], '0987654321'*25000 + '\0')
+ self.assertEqual(m.message['newtup'][3], 'atext\0')
+ messages.expect_commit()
+
+
+
+ def test_default_modes(self):
+ cur = self.conn.cursor()
+
+ cur.execute("INSERT INTO test_text(cola, colb) VALUES('sample1', E'sam\\160le2\\n')")
+ cur.execute("INSERT INTO test_binary values (decode('ff','hex'));")
+ self.conn.commit()
+
+ messages = self.get_changes()
+
+ messages.expect_startup()
+
+ # consume the insert
+ messages.expect_begin()
+ messages.expect_row_meta()
+
+ # The values of the two text fields will be the original text,
+ # returned unchanged, but the escapes in the second one will
+ # have been decoded, so it'll have a literal newline in it
+ # and the octal escape decoded.
+ m = messages.expect_insert()
+ self.assertEqual(m.message['newtup'][0], 'sample1\0')
+ self.assertEqual(m.message['newtup'][1], 'sample2\n\0')
+
+ messages.expect_row_meta()
+ m = messages.expect_insert()
+
+ # While this is a bytea field, we didn't negotiate binary or send/recv
+ # mode with the server, so what we'll receive is the hex-encoded text
+ # representation of the bytea value as as text-format literal.
+ self.assertEqual(m.message['newtup'][0], '\\xff\0')
+
+ messages.expect_commit()
+
+if __name__ == '__main__':
+ unittest.main()
--
2.1.0
Hi,
On 2015-11-02 20:17:21 +0800, Craig Ringer wrote:
I'd like to submit pglogical_output for inclusion in the 9.6 series as
a contrib.
Cool!
See the README.md and DESIGN.md in the attached patch for details on
the plugin. I will follow up with a summary in a separate mail, along
with a few points I'd value input on or want to focus discussion on.
Sounds to me like at least a portion of this should be in sgml, either
in a separate contrib page or in the logical decoding section.
A quick readthrough didn't have a separate description of the
"sub-protocol" in which changes and such are encoded - I think we're
going to need that.
I anticipate that I'll be following up with a few tweaks, but at this
point the plugin is feature-complete, documented and substantially
ready for inclusion as a contrib.
There's a bunch of changes that are hinted at in the files in various
places. Could you extract the ones you think need to be fixed before
integration see in some central place (or just an email)?
Regards,
Andres
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 2 November 2015 at 20:35, Andres Freund <andres@anarazel.de> wrote:
On 2015-11-02 20:17:21 +0800, Craig Ringer wrote:
See the README.md and DESIGN.md in the attached patch for details on
the plugin. I will follow up with a summary in a separate mail, along
with a few points I'd value input on or want to focus discussion on.Sounds to me like at least a portion of this should be in sgml, either
in a separate contrib page or in the logical decoding section.
Yes, I think so. Before rewriting to SGML I wanted to solicit opinions
on what should be hidden away in a for-developers README file and what
parts deserve exposure in the public docs.
A quick readthrough didn't have a separate description of the
"sub-protocol" in which changes and such are encoded - I think we're
going to need that.
It didn't quite make the first cut as I have to make a couple of edits
to reflect late changes. I should be able to follow up with that later
today.
The protocol design documentation actually predates the plugin its
self, though it saw a few changes where it became clear something
wouldn't work as envisioned. It's been quite a pleasure starting with
a detailed design, then implementing it.
There's a bunch of changes that are hinted at in the files in various
places. Could you extract the ones you think need to be fixed before
integration see in some central place (or just an email)?
Yep, writing that up at the moment. I didn't want to make the initial
post too verbose.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 2 November 2015 at 20:17, Craig Ringer <craig@2ndquadrant.com> wrote:
Hi all
I'd like to submit pglogical_output for inclusion in the 9.6 series as
a contrib.
A few points are likely to come up in anything but the most cursory
examination of the patch.
The README alludes to protocol docs that aren't in the tree. A
followup will add them shortly, they just need a few tweaks.
There are pg_regress tests, but they're limited. The output plugin
uses the binary output mode, and pg_regress doesn't play well with
that at all. Timestamps, XIDs, LSNs, etc are embedded in the output.
Also, pglogical its self emits LSNs and timestamps in commit messages.
Some things, like the startup message, are likely to contain variable
data in future too. So we can't easily do a "dumb" comparison of
expected output.
That's why the bulk of the tests in test/ are in Python, using
psycopg2. Python and psycopg2 were used partly because of the
excellent work done by Oleksandr Shulgin at Zalando
(https://github.com/zalando/psycopg2/tree/feature/replication-protocol,
https://github.com/psycopg/psycopg2/pull/322) which means we can
connect to the walsender and consume the replication protocol, rather
than relying only on the SQL interfaces. Both are supported, and only
the SQL interface is used by default. It also means the tests can have
logic to validate the protocol stream, examining it message by message
to ensure it's exactly what's expected. Rather than a diff where two
lines of binary gibberish don't match, you get a specific error. Of
course, I'm aware that the buildfarm animals aren't required to have
Python, let alone a patched psycopg2, so we can't rely on these as
smoketests. That's why the pg_regress tests are there too.
There another extension inside it, in
contrib/pglogical_output/examples/hooks . I'm not sure if this should
be separated out into a separate contrib/ since it's very tightly
coupled to pglogical_output. Its purpose is to expose the hooks from
pglogical_output to SQL, so that they can be implemented by plpgsql or
whatever, instead of having to be C functions. It's not integrated
into pglogical_output proper because I see this as mainly a test and
prototyping facility. It's necessary to have this in order for the
unit tests to cover filtering and hooks, but most practical users will
never want or need it. So I'd rather not integrate it into
pglogical_output proper.
pglogical_output has headers, and it installs them into Pg's include/
tree at install time. This is not something that prior contribs have
done, so there's no policy for it as yet. The reason for doing so is
that the output plugin exposes a hooks API so that it can be reused by
different clients with different needs, rather than being tightly
coupled to just one downstream user. For example, it makes no
assumptions about things like what replication origin names mean -
keeping with the design of replication origins, which just provide
mechanism without policy. That means that the client needs to tell the
output plugin how to filter transactions if it wants to do selective
replication on a node-by-node basis. Similarly, there's no built-in
support for selective replication on a per-table basis, just a hook
you can implement. So clients can provide their own policy for how to
decide what tables to replicate. When we're calling hooks for each and
every row we really want a C function pointer so we can avoid the need
to go through the fmgr each time, and so we can pass a `struct
Relation` and bypass the need for catalog lookups. That sort of thing.
Table metadata is sent for each row. It really needs to be sent once
for each consecutive series of rows for the same table. Some care is
required to make sure it's invalidated and re-sent when the table
structure changes mid-series. So that's a pending change. It's
important for efficiency, but pretty isolated and doesn't make the
plugin less useful otherwise, so I thought it could wait.
Sending the whole old tuple is not yet supported, per the fixme in
pglogical_write_update . It should really be a TODO, since to support
this we really need a way to keep track of replica identity for a
table, but also WAL-log the whole old tuple. (ab)using REPLICA
IDENTITY FULL to log the old tuple means we lose information about
what the real identity key is. So this is more of a wanted future
feature, and I'll change it to a TODO.
I'd like to delay some ERROR messages until after the startup
parameters are sent. That way the client can see more info about the
server's configuration, version, capabilities, etc, and possibly
reconnect with acceptable settings. Because a logical decoding plugin
isn't allowed to generate input during its startup callback, though,
this could mean indefinitely delaying an error until the upstream does
some work that results in a decoding callback. So for now errors on
protocol mismatches, etc, are sent immediately.
Text encoding names are compared byte-wise. They should be looked up
in the catalogs and compared properly. Just not done yet.
I think those are the main points.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 2 November 2015 at 20:17, Craig Ringer <craig@2ndquadrant.com> wrote:
Hi all
I'd like to submit pglogical_output for inclusion in the 9.6 series as
a contrib.
Here's the protocol documentation discussed in the README. It's
asciidoc at the moment, so it can be formatted into something with
readable tables.
If everyone thinks it's reasonable to document the pglogical protocol
as part of the SGML docs then it can be converted. Since the walsender
protocol is documented in the public SGML docs it probably should be,
it's just a matter of deciding what goes where.
Thanks to Darko Milojković for the asciidoc conversion. All errors are
likely to be my edits.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
Attachments:
On Mon, Nov 2, 2015 at 1:17 PM, Craig Ringer <craig@2ndquadrant.com> wrote:
Hi all
I'd like to submit pglogical_output for inclusion in the 9.6 series as
a contrib.
Yay, that looks pretty advanced! :-)
Still digesting...
--
Alex
On 11/2/15 8:36 AM, Craig Ringer wrote:
Here's the protocol documentation discussed in the README. It's
asciidoc at the moment, so it can be formatted into something with
readable tables.
Is this by chance up on github? It'd be easier to read the final output
there than the raw asciidoctor. ;)
Great work on this!
--
Jim Nasby, Data Architect, Blue Treble Consulting, Austin TX
Experts in Analytics, Data Architecture and PostgreSQL
Data in Trouble? Get it in Treble! http://BlueTreble.com
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 3 November 2015 at 02:58, Jim Nasby <Jim.Nasby@bluetreble.com> wrote:
On 11/2/15 8:36 AM, Craig Ringer wrote:
Here's the protocol documentation discussed in the README. It's
asciidoc at the moment, so it can be formatted into something with
readable tables.Is this by chance up on github? It'd be easier to read the final output
there than the raw asciidoctor. ;)
Not yet, no. Should be able to format it to PDF, HTML, etc if needed though.
Depending on the consensus here, I'm expecting the protocol docs will
likely get turned into a plaintext formatted README in the source
tree, or into SGML docs.
The rest are in rather more readable Markdown form at this point,
again pending opinions on where they should live - in the public SGML
docs or in-tree READMEs.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 3 November 2015 at 16:41, Craig Ringer <craig@2ndquadrant.com> wrote:
On 3 November 2015 at 02:58, Jim Nasby <Jim.Nasby@bluetreble.com> wrote:
On 11/2/15 8:36 AM, Craig Ringer wrote:
Here's the protocol documentation discussed in the README. It's
asciidoc at the moment, so it can be formatted into something with
readable tables.Is this by chance up on github? It'd be easier to read the final output
there than the raw asciidoctor. ;)Not yet, no. Should be able to format it to PDF, HTML, etc if needed though.
In fact, I'll just put a PDF up shortly.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 3 November 2015 at 02:58, Jim Nasby <Jim.Nasby@bluetreble.com> wrote:
On 11/2/15 8:36 AM, Craig Ringer wrote:
Here's the protocol documentation discussed in the README. It's
asciidoc at the moment, so it can be formatted into something with
readable tables.Is this by chance up on github? It'd be easier to read the final output
there than the raw asciidoctor. ;)
HTML for the protocol documentation attached.
The docs are being converted to SGML at the moment.
I'll be pushing an update of pglogical_output soon. Patch in a followup
mail.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
Attachments:
On 2015/11/02 23:36, Craig Ringer wrote:
On 2 November 2015 at 20:17, Craig Ringer <craig@2ndquadrant.com> wrote:
Hi all
I'd like to submit pglogical_output for inclusion in the 9.6 series as
a contrib.Here's the protocol documentation discussed in the README. It's
asciidoc at the moment, so it can be formatted into something with
readable tables.
Kudos! From someone who doesn't really read wire protocol docs a lot, this
was such an enlightening read.
Thanks,
Amit
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Hi all
Here's an updated pglogical_output patch.
Selected changes since v1:
- add json protocol output support
- fix encoding comparisons to use parsed encoding not string name
- import protocol documentation
- significantly expand pg_regress tests
- move pglogical_output_plhooks to a top-level contrib
- remove 9.4 compatibility
Full diff since last version at
https://github.com/2ndQuadrant/postgres/compare/pglogical-output-v1...2ndQuadrant:pglogical-output-v2
Notably, it now has support for a basic json-format text-based protocol as
well as the main binary protocol. Mostly for debugging, testing and
diagnostics.
I've used the newly added json support in pglogical_output to significantly
expand the pg_regress tests, since the json mode can support text-mode
decoding in the SQL interface to logical decoding, where the lsn, xid, etc
can be skipped.
I've removed the Python based test framework. Test coverage in-tree is
reduced as a result, with no coverage of the details of the binary
protocol, no coverage of walsender based use, etc. I'll have a look at
whether it'll be practical to convert the tests to Perl code driving
pg_recvlogical as a co-process but don't think evaluation should wait on
this.
I've also removed the 9.4 backward compatibility code from the version
submitted for 9.6.
Docs conversion to SGML is still pending/WIP.
I've been unable to find a reasonable way to send the startup message
before raising an ERROR when there's an issue with parameter/protocol
negotiation. Advice/suggestions appreciated. The main issue is that the
setup callback can't write to the protocol stream since in walsender mode
the walsender isn't ready for writes yet. (Otherwise we could just write a
record with InvalidXLogRecPtr, etc). Delaying the startup msg to the first
begin callback works, as is done now. But delaying an error would involve
allowing the begin callback to complete, then ERRORing at the *next*
callback, which could be anything. Very ugly for what should be an uncommon
case. So for now ERRORs are immediate, even though that makes negotiation
much more difficult. Ideas welcomed.
(Cc'd a few people who expressed interest)
Attachments:
0001-pglogical_output-v2.patchtext/x-patch; charset=UTF-8; name=0001-pglogical_output-v2.patchDownload
From da5fe22500d3f86cc688057055a2a47936eb806d Mon Sep 17 00:00:00 2001
From: Craig Ringer <craig@2ndquadrant.com>
Date: Mon, 2 Nov 2015 19:34:21 +0800
Subject: [PATCH] pglogical_output v2
A general purpose output plugin for PostgreSQL logical replication
Changes since v1:
- add json protocol output support
- fix encoding comparisons to use parsed encoding not string name
- Report datum field encoding and database_encoding separately
- import protocol documentation
- README updates, move some info from protocol doc to README
- send startup msg as List not buffer
- significantly expand pg_regress tests
- move pglogical_output_plhooks to a top-level contrib
- remove 9.4 compatibility
- remove Python tests
---
contrib/Makefile | 2 +
contrib/pglogical_output/.gitignore | 6 +
contrib/pglogical_output/Makefile | 27 +
contrib/pglogical_output/README.md | 524 ++++++++++++++++++++
contrib/pglogical_output/doc/.gitignore | 1 +
contrib/pglogical_output/doc/DESIGN.md | 124 +++++
contrib/pglogical_output/doc/protocol.txt | 546 +++++++++++++++++++++
contrib/pglogical_output/expected/basic_json.out | 108 ++++
contrib/pglogical_output/expected/basic_native.out | 99 ++++
.../pglogical_output/expected/encoding_json.out | 59 +++
contrib/pglogical_output/expected/hooks_json.out | 137 ++++++
contrib/pglogical_output/expected/hooks_native.out | 104 ++++
.../pglogical_output/expected/params_native.out | 118 +++++
.../pglogical_output/expected/params_native_1.out | 94 ++++
contrib/pglogical_output/expected/pre_clean.out | 10 +
contrib/pglogical_output/pglogical_config.c | 499 +++++++++++++++++++
contrib/pglogical_output/pglogical_config.h | 55 +++
contrib/pglogical_output/pglogical_hooks.c | 232 +++++++++
contrib/pglogical_output/pglogical_hooks.h | 22 +
contrib/pglogical_output/pglogical_output.c | 537 ++++++++++++++++++++
contrib/pglogical_output/pglogical_output.h | 105 ++++
contrib/pglogical_output/pglogical_output/README | 7 +
contrib/pglogical_output/pglogical_output/hooks.h | 72 +++
contrib/pglogical_output/pglogical_proto.c | 49 ++
contrib/pglogical_output/pglogical_proto.h | 57 +++
contrib/pglogical_output/pglogical_proto_json.c | 198 ++++++++
contrib/pglogical_output/pglogical_proto_json.h | 32 ++
contrib/pglogical_output/pglogical_proto_native.c | 494 +++++++++++++++++++
contrib/pglogical_output/pglogical_proto_native.h | 37 ++
contrib/pglogical_output/regression.conf | 2 +
contrib/pglogical_output/sql/basic_json.sql | 14 +
contrib/pglogical_output/sql/basic_native.sql | 27 +
contrib/pglogical_output/sql/basic_setup.sql | 62 +++
contrib/pglogical_output/sql/basic_teardown.sql | 4 +
contrib/pglogical_output/sql/encoding_json.sql | 58 +++
contrib/pglogical_output/sql/hooks_json.sql | 28 ++
contrib/pglogical_output/sql/hooks_native.sql | 48 ++
contrib/pglogical_output/sql/hooks_setup.sql | 37 ++
contrib/pglogical_output/sql/hooks_teardown.sql | 10 +
contrib/pglogical_output/sql/params_native.sql | 95 ++++
contrib/pglogical_output/sql/pre_clean.sql | 10 +
contrib/pglogical_output_plhooks/.gitignore | 1 +
contrib/pglogical_output_plhooks/Makefile | 13 +
.../README.pglogical_output_plhooks | 160 ++++++
.../pglogical_output_plhooks--1.0.sql | 89 ++++
.../pglogical_output_plhooks.c | 414 ++++++++++++++++
.../pglogical_output_plhooks.control | 4 +
47 files changed, 5431 insertions(+)
create mode 100644 contrib/pglogical_output/.gitignore
create mode 100644 contrib/pglogical_output/Makefile
create mode 100644 contrib/pglogical_output/README.md
create mode 100644 contrib/pglogical_output/doc/.gitignore
create mode 100644 contrib/pglogical_output/doc/DESIGN.md
create mode 100644 contrib/pglogical_output/doc/protocol.txt
create mode 100644 contrib/pglogical_output/expected/basic_json.out
create mode 100644 contrib/pglogical_output/expected/basic_native.out
create mode 100644 contrib/pglogical_output/expected/encoding_json.out
create mode 100644 contrib/pglogical_output/expected/hooks_json.out
create mode 100644 contrib/pglogical_output/expected/hooks_native.out
create mode 100644 contrib/pglogical_output/expected/params_native.out
create mode 100644 contrib/pglogical_output/expected/params_native_1.out
create mode 100644 contrib/pglogical_output/expected/pre_clean.out
create mode 100644 contrib/pglogical_output/pglogical_config.c
create mode 100644 contrib/pglogical_output/pglogical_config.h
create mode 100644 contrib/pglogical_output/pglogical_hooks.c
create mode 100644 contrib/pglogical_output/pglogical_hooks.h
create mode 100644 contrib/pglogical_output/pglogical_output.c
create mode 100644 contrib/pglogical_output/pglogical_output.h
create mode 100644 contrib/pglogical_output/pglogical_output/README
create mode 100644 contrib/pglogical_output/pglogical_output/hooks.h
create mode 100644 contrib/pglogical_output/pglogical_proto.c
create mode 100644 contrib/pglogical_output/pglogical_proto.h
create mode 100644 contrib/pglogical_output/pglogical_proto_json.c
create mode 100644 contrib/pglogical_output/pglogical_proto_json.h
create mode 100644 contrib/pglogical_output/pglogical_proto_native.c
create mode 100644 contrib/pglogical_output/pglogical_proto_native.h
create mode 100644 contrib/pglogical_output/regression.conf
create mode 100644 contrib/pglogical_output/sql/basic_json.sql
create mode 100644 contrib/pglogical_output/sql/basic_native.sql
create mode 100644 contrib/pglogical_output/sql/basic_setup.sql
create mode 100644 contrib/pglogical_output/sql/basic_teardown.sql
create mode 100644 contrib/pglogical_output/sql/encoding_json.sql
create mode 100644 contrib/pglogical_output/sql/hooks_json.sql
create mode 100644 contrib/pglogical_output/sql/hooks_native.sql
create mode 100644 contrib/pglogical_output/sql/hooks_setup.sql
create mode 100644 contrib/pglogical_output/sql/hooks_teardown.sql
create mode 100644 contrib/pglogical_output/sql/params_native.sql
create mode 100644 contrib/pglogical_output/sql/pre_clean.sql
create mode 100644 contrib/pglogical_output_plhooks/.gitignore
create mode 100644 contrib/pglogical_output_plhooks/Makefile
create mode 100644 contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
diff --git a/contrib/Makefile b/contrib/Makefile
index bd251f6..028fd9a 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -35,6 +35,8 @@ SUBDIRS = \
pg_stat_statements \
pg_trgm \
pgcrypto \
+ pglogical_output \
+ pglogical_output_plhooks \
pgrowlocks \
pgstattuple \
postgres_fdw \
diff --git a/contrib/pglogical_output/.gitignore b/contrib/pglogical_output/.gitignore
new file mode 100644
index 0000000..2322e13
--- /dev/null
+++ b/contrib/pglogical_output/.gitignore
@@ -0,0 +1,6 @@
+pglogical_output.so
+results/
+regression.diffs
+tmp_install/
+tmp_check/
+log/
diff --git a/contrib/pglogical_output/Makefile b/contrib/pglogical_output/Makefile
new file mode 100644
index 0000000..766cab4
--- /dev/null
+++ b/contrib/pglogical_output/Makefile
@@ -0,0 +1,27 @@
+MODULE_big = pglogical_output
+PGFILEDESC = "pglogical_output - logical replication output plugin"
+
+OBJS = pglogical_output.o pglogical_hooks.o pglogical_config.o \
+ pglogical_proto.o pglogical_proto_native.o \
+ pglogical_proto_json.o
+
+REGRESS = pre_clean params_native basic_native hooks_native basic_json hooks_json encoding_json
+
+
+subdir = contrib/pglogical_output
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# 'make installcheck' disabled when building in-tree because these tests
+# require "wal_level=logical", which typical installcheck users do not have
+# (e.g. buildfarm clients).
+installcheck:
+ ;
+
+EXTRA_INSTALL += contrib/pglogical_output_plhooks
+EXTRA_REGRESS_OPTS += --temp-config=./regression.conf
+
+install: all
+ $(MKDIR_P) '$(DESTDIR)$(includedir)'/pglogical_output
+ $(INSTALL_DATA) pglogical_output/hooks.h '$(DESTDIR)$(includedir)'/pglogical_output
diff --git a/contrib/pglogical_output/README.md b/contrib/pglogical_output/README.md
new file mode 100644
index 0000000..4427175
--- /dev/null
+++ b/contrib/pglogical_output/README.md
@@ -0,0 +1,524 @@
+# `pglogical` Output Plugin
+
+This is the [logical decoding](http://www.postgresql.org/docs/current/static/logicaldecoding.html)
+[output plugin](http://www.postgresql.org/docs/current/static/logicaldecoding-output-plugin.html)
+for `pglogical`. Its purpose is to extract a change stream from a PostgreSQL
+database and send it to a client over a network connection using a
+well-defined, efficient protocol that multiple different applications can
+consume.
+
+The primary purpose of `pglogical_output` is to supply data to logical
+streaming replication solutions, but any application can potentially use its
+data stream. The output stream is designed to be compact and fast to decode,
+and the plugin supports upstream filtering of data so that only the required
+information is sent.
+
+Only one database is replicated, rather than the whole PostgreSQL install. A
+subset of that database may be selected for replication, currently based on
+table and on replication origin. Filtering by a WHERE clause can be supported
+easily in future.
+
+No triggers are required to collect the change stream and no external ticker or
+other daemon is required. It's accumulated using
+[replication slots](http://www.postgresql.org/docs/current/static/logicaldecoding-explanation.html#AEN66446),
+as supported in PostgreSQL 9.4 or newer, and sent on top of the
+[PostgreSQL streaming replication protocol](http://www.postgresql.org/docs/current/static/protocol-replication.html).
+
+Unlike block-level ("physical") streaming replication, the change stream from
+the `pglogical` output plugin is compatible across different PostgreSQL
+versions and can even be consumed by non-PostgreSQL clients.
+
+Becuse logical decoding is used, only the changed rows are sent on the wire.
+There's no index change data, no vacuum activity, etc transmitted.
+
+The use of a replication slot means that the change stream is reliable and
+crash-safe. If the client disconnects or crashes it can reconnect and resume
+replay from the last message that client processed. Server-side changes that
+occur while the client is disconnected are accumulated in the queue to be sent
+when the client reconnects. This reliability also means that server-side
+resources are consumed whether or not a client is connected.
+
+# Why another output plugin?
+
+See [`DESIGN.md`](DESIGN.md) for a discussion of why using one of the existing
+generic logical decoding output plugins like `wal2json` to drive a logical
+replication downstream isn't ideal. It's mostly about speed.
+
+# Architecture and high level interaction
+
+The output plugin is loaded by a PostgreSQL walsender process when a client
+connects to PostgreSQL using the PostgreSQL wire protocol with connection
+option `replication=database`, then uses
+[the `CREATE_REPLICATION_SLOT ... LOGICAL ...` or `START_REPLICATION SLOT ... LOGICAL ...` commands](http://www.postgresql.org/docs/current/static/logicaldecoding-walsender.html) to start streaming changes. (It can also be used via
+[SQL level functions](http://www.postgresql.org/docs/current/static/logicaldecoding-sql.html)
+over a non-replication connection, but this is mainly for debugging purposes).
+
+The client supplies parameters to the `START_REPLICATION SLOT ... LOGICAL ...`
+command to specify the version of the `pglogical` protocol it supports,
+whether it wants binary format, etc.
+
+The output plugin processes the connection parameters and the connection enters
+streaming replication protocol mode, sometimes called "COPY BOTH" mode because
+it's based on the protocol used for the `COPY` command. PostgreSQL then calls
+functions in this plugin to send it a stream of transactions to decode and
+translate into network messages. This stream of changes continues until the
+client disconnects.
+
+The only client-to-server interaction after startup is the sending of periodic
+feedback messages that allow the replication slot to discard no-longer-needed
+change history. The client *must* send feedback, otherwise `pg_xlog` on the
+server will eventually fill up and the server will stop working.
+
+
+# Usage
+
+The overall flow of client/server interaction is:
+
+* Client makes PostgreSQL fe/be protocol connection to server
+ * Connection options must include `replication=database` and `dbname=[...]` parameters
+ * The PostgreSQL client library can be `libpq` or anything else that supports the replication sub-protocol
+ * The same mechanisms are used for authentication and protocol encryption as for a normal non-replication connection
+* [Client issues `IDENTIFY_SYSTEM`
+ * Server responds with a single row containing system identity info
+* Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'` if it's setting up for the first time
+ * Server responds with success info and a snapshot identifier
+ * Client may at this point use the snapshot identifier on other connections while leaving this one idle
+* Client issues `START_REPLICATION SLOT slotname LOGICAL 0/0 (...options...)` to start streaming, which loops:
+ * Server emits `pglogical` message block encapsulated in a replication protocol `CopyData` message
+ * Client receives and unwraps message, then decodes the `pglogical` message block
+ * Client intermittently sends a standby status update message to server to confirm replay
+* ... until client sends a graceful connection termination message on the fe/be protocol level or the connection is broken
+
+ The details of `IDENTIFY_SYSTEM`, `CREATE_REPLICATION_SLOT` and `START_REPLICATION` are discussed in the [replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html) and will not be repeated here.
+
+## Make a replication connection
+
+To use the `pglogical` plugin you must first establish a PostgreSQL FE/BE
+protocol connection using the client library of your choice, passing
+`replication=database` as one of the connection parameters. `database` is a
+literal string and is not replaced with the database name; instead the database
+name is passed separately in the usual `dbname` parameter. Note that
+`replication` is not a GUC (configuration parameter) and may not be passed in
+the `options` parameter on the connection, it's a top-level parameter like
+`user` or `dbname`.
+
+Example connection string for `libpq`:
+
+ 'user=postgres replication=database sslmode=verify-full dbname=mydb'
+
+The plug-in name to pass on logical slot creation is `'pglogical'`.
+
+Details are in the replication protocol docs.
+
+## Get system identity
+
+If required you can use the `IDENTIFY_SYSTEM` command, which reports system
+information:
+
+ systemid | timeline | xlogpos | dbname | dboid
+ ---------------------+----------+-----------+--------+-------
+ 6153224364663410513 | 1 | 0/C429C48 | testd | 16385
+ (1 row)
+
+Details in the replication protocol docs.
+
+## Create the slot if required
+
+If your application creates its own slots on first use and hasn't previously
+connected to this database on this system you'll need to create a replication
+slot. This keeps track of the client's replay state even while it's disconnected.
+
+The slot name may be anything your application wants up to a limit of 63
+characters in length. It's strongly advised that the slot name clearly identify
+the application and the host it runs on.
+
+Pass `pglogical` as the plugin name.
+
+e.g.
+
+ CREATE_REPLICATION_SLOT "reporting_host_42" LOGICAL "pglogical";
+
+`CREATE_REPLICATION_SLOT` returns a snapshot identifier that may be used with
+[`SET TRANSACTION SNAPSHOT`](http://www.postgresql.org/docs/current/static/sql-set-transaction.html)
+to see the database's state as of the moment of the slot's creation. The first
+change streamed from the slot will be the change immediately after this
+snapshot was taken. The snapshot is useful when cloning the initial state of a
+database being replicted. Applications that want to see the change stream
+going forward, but don't care about the initial state, can ignore this. The
+snapshot is only valid as long as the connection that issued the
+`CREATE_REPLICATION_SLOT` remains open and has not run another command.
+
+## Send replication parameters
+
+The client now sends:
+
+ START_REPLICATION SLOT "the_slot_name" LOGICAL (
+ 'Expected_encoding', 'UTF8',
+ 'Max_proto_major_version', '1',
+ 'Min_proto_major_version', '1',
+ ...moreparams...
+ );
+
+to start replication.
+
+The parameters are very important for ensuring that the plugin accepts
+the replication request and streams changes in the expected form. `pglogical`
+parameters are discussed in the separate `pglogical` protocol documentation.
+
+## Process the startup message
+
+`pglogical`'s output plugin will send a `CopyData` message containing its
+startup message as the first protocol message. This message contains a
+set of key/value entries describing the capabilities of the upstream output
+plugin, its version and the Pg version, the tuple format options selected,
+etc.
+
+The downstream client may choose to cleanly close the connection and disconnect
+at this point if it doesn't like the reply. It might then inform the user
+or reconnect with different parameters based on what it learned from the
+first connection's startup message.
+
+## Consume the change stream
+
+`pglogical`'s output plugin now sends a continuous series of `CopyData`
+protocol messages, each of which encapsulates a `pglogical` protocol message
+as documented in the separate protocol docs.
+
+These messages provide information about transaction boundaries, changed
+rows, etc.
+
+The stream continues until the client disconnects, the upstream server is
+restarted, the upstream walsender is terminated by admin action, there's
+a network issue, or the connection is otherwise broken.
+
+The client should send periodic feedback messages to the server to acknowledge
+that it's replayed to a given point and let the server release the resources
+it's holding in case that change stream has to be replayed again. See
+["Hot standby feedback message" in the replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html)
+for details.
+
+## Disconnect gracefully
+
+Disconnection works just like any normal client; you use your client library's
+usual method for closing the connection. No special action is required before
+disconnection, though it's usually a good idea to send a final standby status
+message just before you disconnect.
+
+# Tests
+
+The `pg_regress` tests check invalid parameter handling and basic
+functionality. They're intended for use by the buildfarm using an in-tree
+`make check`, but may also be run with an out-of-tree PGXS build against an
+existing PostgreSQL install using `make USE_PGXS=1 clean installcheck`.
+
+The tests may fail on installations that are not utf-8 encoded because the
+payloads of the binary protocol output will have text in different encodings,
+which aren't visible to psql as text to be decoded. Avoiding anything except
+7-bit ascii in the tests *should* prevent the problem.
+
+# Changeset forwarding
+
+It's possible to use `pglogical_output` to cascade replication between multiple
+PostgreSQL servers, in combination with an appropriate client to apply the
+changes to the downstreams.
+
+There are three forwarding modes:
+
+* Forward everything. Transactions are replicated whether they were made directly
+ on the immediate upstream or some other node upstream of it. This is the only
+ option when running on 9.4. All rows from transactions are sent.
+
+ Selected by setting `forward_changesets` to true (default) and not setting a
+ row or transaction filter hook.
+
+* No forwarding. Only transactions applied immediately on the upstream node are
+ forwarded. Transactions with any non-local origin are skipped. All rows from
+ locally originated transactions are sent.
+
+ Selected by setting `forward_changesets` to false. Remember to confirm by
+ checking the startup reply message.
+
+* Filtered forwarding. Transactions are replicated unless a client-supplied
+ transaction filter hook says to skip this transaction. Row changes are
+ replicated unless the client-supplied row filter hook (if provided) says to
+ skip that row.
+
+ Selected by setting `forward_changesets` to `true` and installing a
+ transaction and/or row filter hook (see "hooks").
+
+If the upstream server is 9.5 or newer and `forward_changesets` is enabled, the
+server will enable changeset origin information. It will set
+`forward_changeset_origins` to true in the startup reply message to indicate
+this. It will then send changeset origin messages after the `BEGIN` for each
+transaction, per the protocol documentation. Origin messages are omitted for
+transactions originating directly on the immediate upstream to save bandwidth.
+If `forward_changeset_origins` is true then transactions without an origin are
+always from the immediate upstream that’s running the decoding plugin.
+
+Note that changeset forwarding may be forced to on if not requested by some
+servers, so the client _should_ check the forward_changesets and
+`forward_changeset_origins` params in the startup reply message. In particular,
+9.4 servers force changeset forwarding on, but never forward replication
+origins. This means you cannot use 9.4 for mutual replication as it’ll create
+an infinite loop.
+
+Clients may use this facility to form arbitrarily complex topologies when
+combined with hooks to determine which transactions are forwarded. An obvious
+case is bi-directional (mutual) replication.
+
+# Selective replication
+
+By specifying a row filter hook it's possible to filter the replication stream
+server-side so that only a subset of changes is replicated.
+
+
+# Hooks
+
+`pglogical_output` exposes a number of extension points where applications can
+modify or override its behaviour.
+
+All hooks are called in their own memory context, which lasts for the duration
+of the logical decoding session. They may switch to longer lived contexts if
+needed, but are then responsible for their own cleanup.
+
+## Hook setup function
+
+The downstream must specify the fully-qualified name of a SQL-callable function
+on the server as the value of the `hooks.setup_function` client parameter.
+The SQL signature of this function is
+
+ CREATE OR REPLACE FUNCTION funcname(hooks internal, memory_context internal)
+ RETURNS void STABLE
+ LANGUAGE c AS 'MODULE_PATHNAME';
+
+Permissions are checked. This function must be callable by the user that the
+output plugin is running as. The function name *must* be schema-qualified and is
+parsed like any other qualified identifier.
+
+The function receives a pointer to a newly allocated structure of hook function
+pointers to populate as its first argument. The function must not free the
+argument.
+
+If the hooks need a private data area to store information across calls, the
+setup function should get the `MemoryContext` pointer from the 2nd argument,
+then `MemoryContextAlloc` a struct for the data in that memory context and
+store the pointer to it in `hooks->hooks_private_data`. This will then be
+accessible on future calls to hook functions. It need not be manually freed, as
+the memory context used for logical decoding will free it when it's freed.
+Don't put anything in it that needs manual cleanup.
+
+Each hook has its own C signature (defined below) and the pointers must be
+directly to the functions. Hooks that the client does not wish to set must be
+left null.
+
+An example is provided in `contrib/pglogical_output_plhooks` and the argument
+structs are defined in `pglogical_output/hooks.h`, which is installed into the
+PostgreSQL source tree when the extension is installed.
+
+Each hook that is enabled results in a new startup parameter being emitted in
+the startup reply message. Clients must check for these and must not assume a
+hook was successfully activated because no error is seen.
+
+Hook functions are called in the context of the backend doing logical decoding.
+Except for the startup hook, hooks see the catalog state as it was at the time
+the transaction or row change being examined was made. Access to to non-catalog
+tables is unsafe unless they have the `user_catalog_table` reloption set.
+
+## Startup hook
+
+The startup hook is called when logical decoding starts.
+
+This hook can inspect the parameters passed by the client to the output
+plugin as in_params. These parameters *must not* be modified.
+
+It can add new parameters to the set to be returned to the client in the
+startup parameters message, by appending to List out_params, which is
+initially NIL. Each element must be a `DefElem` with the param name
+as the `defname` and a `String` value as the arg, as created with
+`makeDefElem(...)`. It and its contents must be allocated in the
+logical decoding memory context.
+
+For walsender based decoding the startup hook is called only once, and
+cleanup might not be called at the end of the session.
+
+Multiple decoding sessions, and thus multiple startup hook calls, may happen
+in a session if the SQL interface for logical decoding is being used. In
+that case it's guaranteed that the cleanup hook will be called between each
+startup.
+
+When successfully enabled, the output parameter `hooks.startup_hook_enabled` is
+set to true in the startup reply message.
+
+Unlike the other hooks, this hook sees a snapshot of the database's current
+state, not a time-traveled catalog state. It is safe to access all tables from
+this hook.
+
+## Transaction filter hook
+
+The transaction filter hook can exclude entire transactions from being decoded
+and replicated based on the node they originated from.
+
+It is passed a `const TxFilterHookArgs *` containing:
+
+* The hook argument supplied by the client, if any
+* The `RepOriginId` that this transaction originated from
+
+and must return boolean, where true retains the transaction for sending to the
+client and false discards it. (Note that this is the reverse sense of the low
+level logical decoding transaction filter hook).
+
+The hook function must *not* free the argument struct or modify its contents.
+
+The transaction filter hook is only called on PostgreSQL 9.5 and above. It
+is ignored on 9.4.
+
+Note that individual changes within a transaction may have different origins to
+the transaction as a whole; see "Origin filtering" for more details. If a
+transaction is filtered out, all changes are filtered out even if their origins
+differ from that of the transaction as a whole.
+
+When successfully enabled, the output parameter
+`hooks.transaction_filter_enabled` is set to true in the startup reply message.
+
+## Row filter hook
+
+The row filter hook is called for each row. It is passed information about the
+table, the transaction origin, and the row origin.
+
+It is passed a `const RowFilterHookArgs*` containing:
+
+* The hook argument supplied by the client, if any
+* The `Relation` the change affects
+* The change type - 'I'nsert, 'U'pdate or 'D'elete
+
+It can return true to retain this row change, sending it to the client, or
+false to discard it.
+
+The function *must not* free the argument struct or modify its contents.
+
+Note that it is more efficient to exclude whole transactions with the
+transaction filter hook rather than filtering out individual rows.
+
+When successfully enabled, the output parameter
+`hooks.row_filter_enabled` is set to true in the startup reply message.
+
+## Shutdown hook
+
+The shutdown hook is called when a decoding session ends. You can't rely on
+this hook being invoked reliably, since a replication-protocol walsender-based
+session might just terminate. It's mostly useful for cleanup to handle repeated
+invocations under the SQL interface to logical decoding.
+
+You don't need a hook to free memory you allocated, unless you explicitly
+switched to a longer lived memory context like TopMemoryContext. Memory allocated
+in the hook context will be automatically when the decoding session shuts down.
+
+## Writing hooks in procedural languages
+
+You can write hooks in PL/PgSQL, etc, too, via the `pglogical_output_plhooks`
+adapter extension in `contrib`. They won't perform very well though.
+
+# Limitations
+
+The advantages of logical decoding in general and `pglogical_output` in
+particular are discussed above. There are also some limitations that apply to
+`pglogical_output`, and to Pg's logical decoding in general.
+
+(TODO: move much of this to the main logical decoding docs)
+
+Notably:
+
+## Doesn't replicate DDL
+
+Logical decoding doesn't decode catalog changes directly. So the plugin can't
+just send a `CREATE TABLE` statement when a new table is added.
+
+If the data being decoded is being applied to another PostgreSQL database then
+its table definitions must be kept in sync via some means external to the logical
+decoding plugin its self, such as:
+
+* Event triggers using DDL deparse to capture DDL changes as they happen and write them to a table to be replicated and applied on the other end; or
+* doing DDL management via tools that synchronise DDL on all nodes
+
+## Doesn't replicate global objects/shared catalog changes
+
+PostgreSQL has a number of object types that exist across all databases, stored
+in *shared catalogs*. These include:
+
+* Roles (users/groups)
+* Security labels on users and databases
+
+Such objects cannot be replicated by `pglogical_output`. They're managed with DDL that
+can't be captured within a single database and isn't decoded anyway.
+
+DDL for global object changes must be synchronized via some external means.
+
+## Mostly one-way communication
+
+Per the protocol documentation, the downstream can't send anything except
+replay progress messages to the upstream after replication begins, and can't
+re-initialise replication without a disconnect.
+
+To achieve downstream-to-upstream communication, clients can use a regular
+libpq connection to the upstream then write to tables or call functions.
+Alternately, a separate replication connection in the opposite direction can be
+created by the application to carry information from downstream to upstream.
+
+See "Protocol flow" in the protocol documentation for more information.
+
+## Physical replica failover
+
+Logical decoding cannot follow a physical replication failover because
+replication slot state is not replicated to physical replicas. If you fail over
+to a streaming replica you have to manually reconnect your logical replication
+clients, creating new slots, etc. This is a core PostgreSQL limitation.
+
+Also, there's no built-in way to guarantee that the logical replication slot
+from the failed master hasn't replayed further than the physical streaming
+replica you failed over to. You could recieve changes on your logical decoding
+stream from the old master that never made it to the physical streaming
+replica. This is true (albeit very unlikely) *even if the physical streaming
+replica is synchronous* because PostgreSQL sends the replication data anyway,
+then just delays the commit's visibility on the master. Support for strictly
+ordered standbys would be required in PostgreSQL to avoid this.
+
+To achieve failover with logical replication you cannot mix in physical
+standbys. The logical replication client has to take responsibility for
+maintaining slots on logical replicas intended as failover candidates
+and for ensuring that the furthest-ahead replica is promoted if there is
+more than one.
+
+## Can only replicate complete transactions
+
+Logical decoding can only replicate a transaction after it has committed. This
+usefully skips replication of rolled back transactions, but it also means that
+very large transactions must be completed upstream before they can begin on the
+downstream, adding to replication latency.
+
+## Replicates only one transaction at a time
+
+Logical decoding serializes transactions in commit order, so pglogical_output
+cannot replay interleaved concurrent transactions. This can lead to high latencies
+when big transactions are being replayed, since smaller transactions get queued
+up behind them.
+
+## Unique index required for inserts or updates
+
+To replicate `INSERT`s or `UPDATE`s it is necessary to have a `PRIMARY KEY`
+or a (non-partial, columns-only) `UNIQUE` index on the table, so the table
+has a `REPLICA IDENTITY`. Without that `pglogical_output` doesn't know what
+old key to send to allow the receiver to tell which tuple is being updated.
+
+## UNLOGGED tables aren't replicated
+
+Because `UNLOGGED` tables aren't written to WAL, they aren't replicated by
+logical or physical repliation. You can only replicate `UNLOGGED` tables
+with trigger-based solutions.
+
+## Unchanged fields are often sent in `UPDATE`
+
+Because there's no tracking of dirty/clean fields when a tuple is updated,
+logical decoding can't tell if a given field was changed by an update.
+Unchanged fields can only by identified and omitted if they're a variable
+length TOASTable type and are big enough to get stored out-of-line in
+a TOAST table.
diff --git a/contrib/pglogical_output/doc/.gitignore b/contrib/pglogical_output/doc/.gitignore
new file mode 100644
index 0000000..2874bff
--- /dev/null
+++ b/contrib/pglogical_output/doc/.gitignore
@@ -0,0 +1 @@
+protocol.html
diff --git a/contrib/pglogical_output/doc/DESIGN.md b/contrib/pglogical_output/doc/DESIGN.md
new file mode 100644
index 0000000..05fb4d1
--- /dev/null
+++ b/contrib/pglogical_output/doc/DESIGN.md
@@ -0,0 +1,124 @@
+# Design decisions
+
+Explanations of why things are done the way they are.
+
+## Why does pglogical_output exist when there's wal2json etc?
+
+`pglogical_output` does plenty more than convert logical decoding change
+messages to a wire format and send them to the client.
+
+It handles format negotiations, sender-side filtering using pluggable hooks
+(and the associated plugin handling), etc. The protocol its self is also
+important, and incorporates elements like binary datum transfer that can't be
+easily or efficiently achieved with json.
+
+## Custom binary protocol
+
+Why do we have a custom binary protocol inside the walsender / copy both protocol,
+rather than using a json message representation?
+
+Speed and compactness. It's expensive to create json, with lots of allocations.
+It's expensive to decode it too. You can't represent raw binary in json, and must
+encode it, which adds considerable overhead for some data types. Using the
+obvious, easy to decode json representations also makes it difficult to do
+later enhancements planned for the protocol and decoder, like caching row
+metadata.
+
+The protocol implementation is fairly well encapsulated, so in future it should
+be possible to emit json instead for clients that request it. Right now that's
+not the priority as tools like wal2json already exist for that.
+
+## Column metadata
+
+The output plugin sends metadata for columsn - at minimum, the column names -
+before each row. It will soon be changed to send the data before each row from
+a new, different table, so that streams of inserts from COPY etc don't repeat
+the metadata each time. That's just a pending feature.
+
+The reason metadata must be sent is that the upstream and downstream table's
+attnos don't necessarily correspond. The column names might, and their ordering
+might even be the same, but any column drop or column type change will result
+in a dropped column on one side. So at the user level the tables look the same,
+but their attnos don't match, and if we rely on attno for replication we'll get
+the wrong data in the wrong columns. Not pretty.
+
+That could be avoided by requiring that the downstream table be strictly
+maintained by DDL replication, but:
+
+* We don't want to require DDL replication
+* That won't work with multiple upstreams feeding into a table
+* The initial table creation still won't be correct if the table has dropped
+ columns, unless we (ab)use `pg_dump`'s `--binary-upgrade` support to emit
+ tables with dropped columns, which we don't want to do.
+
+So despite the bandwidth cost, we need to send metadata.
+
+In future a client-negotiated cache is planned, so that clients can announce
+to the output plugin that they can cache metadata across change series, and
+metadata can only be sent when invalidated by relation changes or when a new
+relation is seen.
+
+Support for type metadata is penciled in to the protocol so that clients that
+don't have table definitions at all - like queueing engines - can decode the
+data. That'll also permit type validation sanity checking on the apply side
+with logical replication.
+
+## Hook entry point as a SQL function
+
+The hooks entry point is a SQL function that populates a passed `internal`
+struct with hook function pointers.
+
+The reason for this is that hooks are specified by a remote peer over the
+network. We can't just let the peer say "dlsym() this arbitrary function name
+and call it with these arguments" for fairly obvious security reasons. At bare
+minimum all replication using hooks would have to be superuser-only if we did
+that.
+
+The SQL entry point is only called once per decoding session and the rest of
+the calls are plain C function pointers.
+
+## The startup reply message
+
+The protocol design choices available to `pg_logical` are constrained by being
+contained in the copy-both protocol within the fe/be protocol, running as a
+logical decoding plugin. The plugin has no direct access to the network socket
+and can't send or receive messages whenever it wants, only under the control of
+the walsender and logical decoding framework.
+
+The only opportunity for the client to send data directly to the logical
+decoding plugin is in the `START_REPLICATION` parameters, and it can't send
+anything to the client before that point.
+
+This means there's no opportunity for a multi-way step negotiation between
+client and server. We have to do all the negotiation we're going to in a single
+exchange of messages - the setup parameters and then the replication start
+message. All the client can do if it doesn't like the offer the server makes is
+disconnect and try again with different parameters.
+
+That's what the startup message is for. It reports the plugin's capabilities
+and tells the client which requested options were honoured. This gives the
+client a chance to decide if it's happy with the output plugin's decision
+or if it wants to reconnect and try again with different options. Iterative
+negotiation, effectively.
+
+## Unrecognised parameters MUST be ignored by client and server
+
+To ensure upward and downward compatibility, the output plugin must ignore
+parameters set by the client if it doesn't recognise them, and the client
+must ignore parameters it doesn't recognise in the server's startup reply
+message.
+
+This ensures that older clients can talk to newer servers and vice versa.
+
+For this to work, the server must never enable new functionality such as
+protocol message types, row formats, etc without the client explicitly
+specifying via a startup parameter that it understands the new functionality.
+Everything must be negotiated.
+
+Similarly, a newer client talking to an older server may ask the server to
+enable functionality, but it can't assume the server will actually honour that
+request. It must check the server's startup reply message to see if the server
+confirmed that it enabled the requested functionality. It might choose to
+disconnect and report an error to the user if the server didn't do what it
+asked. This can be important, e.g. when a security-significant hook is
+specified.
diff --git a/contrib/pglogical_output/doc/protocol.txt b/contrib/pglogical_output/doc/protocol.txt
new file mode 100644
index 0000000..0ab41bf
--- /dev/null
+++ b/contrib/pglogical_output/doc/protocol.txt
@@ -0,0 +1,546 @@
+= Pg_logical protocol
+
+pglogical_output defines a libpq subprocotol for streaming tuples, metadata,
+etc, from the decoding plugin to receivers.
+
+This protocol is an inner layer in a stack:
+
+ * tcp or unix sockets
+ ** libpq protocol
+ *** libpq replication subprotocol (COPY BOTH etc)
+ **** pg_logical output plugin => consumer protocol
+
+so clients can simply use libpq's existing replication protocol support,
+directly or via their libpq-wrapper driver.
+
+This is a binary protocol intended for compact representation.
+
+`pglogical_output` also supports a json-based text protocol with json
+representations of the same changesets, supporting all the same hooks etc,
+intended mainly for tracing/debugging/diagnostics. That protocol is not
+discussed here.
+
+== ToC
+
+== Protocol flow
+
+The protocol flow is primarily from upstream walsender/decoding plugin to the
+downstream receiver.
+
+The only information the flows downstream-to-upstream is:
+
+ * The initial parameter list sent to `START_REPLICATION`; and
+ * replay progress messages
+
+We can accept an arbitrary list of params to `START_REPLICATION`. After
+that we have no general purpose channel for information to flow upstream. That
+means we can't do a multi-step negotiation/handshake for determining the
+replication options to use, binary protocol, etc.
+
+The main form of negotiation is the client getting a "take it or leave it" set
+of settings from the server in an initial startup message sent before any
+replication data (see below) and, if it doesn't like them, reconnecting with
+different startup options.
+
+Except for the negotiation via initial parameter list and then startup message
+the protocol flow is the same as any other walsender-based logical replication
+plugin. The data stream is sent in COPY BOTH mode as a series of CopyData
+messages encapsulating replication data, and ends when the client disconnects.
+There's no facility for ending the COPY BOTH mode and returning to the
+walsender command parser to issue new commands. This is a limiation of the
+walsender interface, not pglogical_output.
+
+== Protocol messages
+
+The individual protocol messages are discussed in the following sub-sections.
+Protocol flow and logic comes in the next major section.
+
+Absolutely all top-level protocol messages begin with a message type byte.
+While represented in code as a character, this is a signed byte with no
+associated encoding.
+
+Since the PostgreSQL libpq COPY protocol supplies a message length there’s no
+need for top-level protocol messages to embed a length in their header.
+
+=== BEGIN message
+
+A stream of rows starts with a `BEGIN` message. Rows may only be sent after a
+`BEGIN` and before a `COMMIT`.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**B**’ (0x42)
+|flags|uint8| * 0-3: Reserved, client _must_ ERROR if set and not recognised.
+|lsn|uint64|“final_lsn” in decoding context - currently it means lsn of commit
+|commit time|uint64|“commit_time” in decoding context
+|remote XID|uint32|“xid” in decoding context
+|===
+
+=== Forwarded transaction origin message
+
+The message after the `BEGIN` may be a _forwarded transaction origin_ message
+indicating what upstream node the transaction came from.
+
+Sent if the immediately prior message was a `BEGIN` message, the upstream
+transaction was forwarded from another node, and replication origin forwarding
+is enabled, i.e. `forward_changeset_origins` is `t` in the startup reply
+message.
+
+A "node" could be another host, another DB on the same host, or pretty much
+anything. Whatever origin name is found gets forwarded. The origin identifier
+is of arbitrary and application-defined format. Applications _should_ prefix
+their origin identifier with a fixed application name part, like `bdr_`,
+`myapp_`, etc. It is application-defined what an application does with
+forwarded transactions from other applications.
+
+An origin message with a zero-length origin name indicates that the origin
+could not be identified but was (probably) not the local node. It is
+client-defined what action is taken in this case.
+
+It is a protocol error to send/receive a forwarded transaction origin message
+at any time other than immediately after a `BEGIN` message.
+
+The origin identifier is typically closely related to replication slot names
+and replication origins’ names in an application system.
+
+For more detail see _Changeset Forwarding_ in the README.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**O**’ (0x4f)
+|flags|uint8| * 0-3: Reserved, application _must_ ERROR if set and not recognised
+|origin_lsn|uint64|Log sequence number (LSN, XLogRecPtr) of the transaction’s commit record on its origin node (as opposed to the forwarding node’s commit LSN, which is ‘lsn’ in the BEGIN message)
+|origin_identifier_length|uint8|Length in bytes of origin_identifier
+|origin_identifier|signed char[origin_identifier_length]|An origin identifier of arbitrary, upstream-application-defined structure. _Should_ be text in the same encoding as the upstream database. NULL-terminated. _Should_ be 7-bit ASCII.
+|===
+
+=== COMMIT message
+A stream of rows ends with a `COMMIT` message.
+
+There is no `ROLLBACK` message because aborted transactions are not sent by the
+upstream.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**C**’ (0x43)
+|Flags|uint8| * 0-3: Reserved, client _must_ ERROR if set and not recognised
+|Commit LSN|uint64|commit_lsn in decoding commit decode callback. This is the same value as in the BEGIN message, and marks the end of the transaction.
+|End LSN|uint64|end_lsn in decoding transaction context
+|Commit time|uint64|commit_time in decoding transaction context
+|===
+
+=== INSERT, UPDATE or DELETE message
+
+After a `BEGIN` or metadata message, the downstream should expect to receive
+zero or more row change messages, composed of an insert/update/delete message
+with zero or more tuple fields, each of which has one or more tuple field
+values.
+
+The row’s relidentifier _must_ match that of the most recently preceding
+metadata message. All consecutive row messages must currently have the same
+relidentifier. (_Later extensions to add metadata caching will relax these
+requirements for clients that advertise caching support; see the documentation
+on metadata messages for more detail_).
+
+It is an error to decode rows using metadata received after the row was
+received, or using metadata that is not the most recently received metadata
+revision that still predates the row. I.e. in the sequence M1, R1, R2, M2, R3,
+M4: R1 and R2 must be decoded using M1, and R3 must be decoded using M2. It is
+an error to use M4 to decode any of the rows, to use M1 to decode R3, or to use
+M2 to decode R1 and R2.
+
+Row messages _may not_ arrive except during a transaction as delimited by `BEGIN`
+and `COMMIT` messages. It is an error to receive a row message outside a
+transaction.
+
+Any unrecognised tuple type or tuple part type is an error on the downstream
+that must result in a client disconnect and error message. Downstreams are
+expected to negotiate compatibility, and upstreams must not add new tuple types
+or tuple field types without negotiation.
+
+The downstream reads rows until the next non-row message is received. There is
+no other end marker or any indication of how many rows to expect in a sequence.
+
+==== Row message header
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**I**’nsert (0x49), ‘**U**’pdate’ (0x55) or ‘**D**’elete (0x44)
+|flags|uint8|Row flags (reserved)
+|relidentifier|uint32|relidentifier that matches the table metadata message sent for this row.
+(_Not present in BDR, which sends nspname and relname instead_)
+|[tuple parts]|[composite]|
+|===
+
+One or more tuple-parts fields follow.
+
+==== Tuple fields
+
+|===
+|Tuple type|signed char|Identifies the kind of tuple being sent.
+
+|tupleformat|signed char|‘**T**’ (0x54)
+|natts|uint16|Number of fields sent in this tuple part.
+(_Present in BDR, but meaning significantly different here)_
+|[tuple field values]|[composite]|
+|===
+
+===== Tuple tupleformat compatibility
+
+Unrecognised _tupleformat_ kinds are a protocol error for the downstream.
+
+==== Tuple field value fields
+
+These message parts describe individual fields within a tuple.
+
+There are two kinds of tuple value fields, abbreviated and full. Which is being
+read is determined based on the first field, _kind_.
+
+Abbreviated tuple value fields are nothing but the message kind:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|kind|signed char| * ‘**n**’ull (0x6e) field
+|===
+
+Full tuple value fields have a length and datum:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|kind|signed char| * ‘**i**’nternal binary (0x62) field
+|length|int4|Only defined for kind = i\|b\|t
+|data|[length]|Data in a format defined by the table metadata and column _kind_.
+|===
+
+===== Tuple field values kind compatibility
+
+Unrecognised field _kind_ values are a protocol error for the downstream. The
+downstream may not continue processing the protocol stream after this
+point**.**
+
+The upstream may not send ‘**i**’nternal or ‘**b**’inary format values to the
+downstream without the downstream negotiating acceptance of such values. The
+downstream will also generally negotiate to receive type information to use to
+decode the values. See the section on startup parameters and the startup
+message for details.
+
+=== Table/row metadata messages
+
+Before sending changed rows for a relation, a metadata message for the relation
+must be sent so the downstream knows the namespace, table name, column names,
+optional column types, etc. A relidentifier field, an arbitrary numeric value
+unique for that relation on that upstream connection, maps the metadata to
+following rows.
+
+A client should not assume that relation metadata will be followed immediately
+(or at all) by rows, since future changes may lead to metadata messages being
+delivered at other times. Metadata messages may arrive during or between
+transactions.
+
+The upstream may not assume that the downstream retains more metadata than the
+one most recent table metadata message. This applies across all tables, so a
+client is permitted to discard metadata for table x when getting metadata for
+table y. The upstream must send a new metadata message before sending rows for
+a different table, even if that metadata was already sent in the same session
+or even same transaction. _This requirement will later be weakened by the
+addition of client metadata caching, which will be advertised to the upstream
+with an output plugin parameter._
+
+Columns in metadata messages are numbered from 0 to natts-1, reading
+consecutively from start to finish. The column numbers do not have to be a
+complete description of the columns in the upstream relation, so long as all
+columns that will later have row values sent are described. The upstream may
+choose to omit columns it doesn’t expect to send changes for in any given
+series of rows. Column numbers are not necessarily stable across different sets
+of metadata for the same table, even if the table hasn’t changed structurally.
+
+A metadata message may not be used to decode rows received before that metadata
+message.
+
+==== Table metadata header
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**R**’ (0x52)
+|flags|uint8| * 0-6: Reserved, client _must_ ERROR if set and not recognised.
+|relidentifier|uint32|Arbitrary relation id, unique for this upstream. In practice this will probably be the upstream table’s oid, but the downstream can’t assume anything.
+|nspnamelength|uint8|Length of namespace name
+|nspname|signed char[nspnamelength]|Relation namespace (null terminated)
+|relnamelength|uint8|Length of relation name
+|relname|char[relname]|Relation name (null terminated)
+|attrs block|signed char|Literal: ‘**A**’ (0x41)
+|natts|uint16|number of attributes
+|[fields]|[composite]|Sequence of ‘natts’ column metadata blocks, each of which begins with a column delimiter followed by zero or more column metadata blocks, each with the same column metadata block header.
+
+This chunked format is used so that new metadata messages can be added without breaking existing clients.
+|===
+
+==== Column delimiter
+
+Each column’s metadata begins with a column metadata header. This comes
+immediately after the natts field in the table metadata header or after the
+last metadata block in the prior column.
+
+It has the same char header as all the others, and the flags field is the same
+size as the length field in other blocks, so it’s safe to read this as a column
+metadata block header.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|blocktype|signed char|‘**C**’ (0x43) - column
+|flags|uint8|Column info flags
+|===
+
+==== Column metadata block header
+
+All column metadata blocks share the same header, which is the same length as a
+column delimiter:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|blocktype|signed char|Identifies the kind of metadata block that follows.
+|blockbodylength|uint16|Length of block in bytes, excluding blocktype char and length field.
+|===
+
+==== Column name block
+
+This block just carries the name of the column, nothing more. It begins with a
+column metadata block, and the rest of the message is the column name.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|[column metadata block header]|[composite]|blocktype = ‘**N**’ (0x4e)
+|colname|char[blockbodylength]|Column name.
+|===
+
+
+==== Column type block
+
+T.B.D.
+
+Not defined in first protocol revision.
+
+Likely to send a type identifier (probably the upstream oid) as a reference to
+a “type info” protocol message to be delivered before. Then we can cache the
+type descriptions and avoid repeating long schemas and names, just using the
+oids.
+
+Needs to have room to handle:
+
+ * built-in core types
+ * extension types (ext version may vary)
+ * enum types (CREATE TYPE … AS ENUM)
+ * range types (CREATE TYPE … AS RANGE)
+ * composite types (CREATE TYPE … AS (...))
+ * custom types (CREATE TYPE ( input = x_in, output = x_out ))
+
+… some of which can be nested
+
+== Startup message
+
+After processing output plugin arguments, the upstream output plugin must send
+a startup message as its first message on the wire. It is a trivial header
+followed by alternating key and value strings represented as null-terminated
+unsigned char strings.
+
+This message specifies the capabilities the output plugin enabled and describes
+the upstream server and plugin. This may change how the client decodes the data
+stream, and/or permit the client to disconnect and report an error to the user
+if the result isn’t acceptable.
+
+If replication is rejected because the client is incompatible or the server is
+unable to satisfy required options, the startup message may be followed by a
+libpq protocol FATAL message that terminates the session. See “Startup errors”
+below.
+
+The parameter names and values are sent as alternating key/value pairs as
+null-terminated strings, e.g.
+
++“key1\0parameter1\0key2\0value2\0”+
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|‘**S**’ (0x53) - startup
+|Startup message version|uint8|Value is always “1”.
+|(parameters)|null-terminated key/value pairs|See table below for parameter definitions.
+|===
+
+=== Startup message parameters
+
+Since all parameter values are sent as strings, the value types given below specify what the value must be reasonably interpretable as.
+
+|===
+|*Key name*|*Value type*|*Description*
+
+|max_proto_version|integer|Newest version of the protocol supported by output plugin.
+|min_proto_version|integer|Oldest protocol version supported by server.
+|proto_format|text|Protocol format requested. native (documented here) or json. Default is native.
+|coltypes|boolean|Column types will be sent in table metadata.
+|pg_version_num|integer|PostgreSQL server_version_num of server, if it’s PostgreSQL. e.g. 090400
+|pg_version|string|PostgreSQL server_version of server, if it’s PostgreSQL.
+|pg_catversion|uint32|Version of the PostgreSQL system catalogs on the upstream server, if it’s PostgreSQL.
+|binary|_set of parameters, specified separately_|See “_the __‘binary’__ parameters_” below, and “_Parameters relating to exchange of binary values_”
+|database_encoding|string|The native text encoding of the database the plugin is running in
+|encoding|string|Field values for textual data will be in this encoding in native protocol text, binary or internal representation. For the native protocol this is currently always the same as `database_encoding`. For text-mode json protocol this is always the same as `client_encoding`.
+|forward_changesets|bool|Specifies that all transactions, not just those originating on the upstream, will be forwarded. See “_Changeset forwarding_”.
+|forward_changeset_origins|bool|Tells the client that the server will send changeset origin information. Independent of forward_changesets. See “_Changeset forwarding_” for details.
+|no_txinfo|bool|Requests that variable transaction info such as XIDs, LSNs, and timestamps be omitted from output. Mainly for tests. Currently ignored for protos other than json.
+|===
+
+
+The ‘binary’ parameter set:
+==
+|===
+|*Key name*|*Value type*|*Description*
+
+|binary.internal_basetypes|boolean|If true, PostgreSQL internal binary representations for row field data may be used for some or all row fields, if here the type is appropriate and the binary compatibility parameters of upstream and downstream match. See binary.want_internal_basetypes in the output plugin parameters for details.
+
+May only be true if _binary.want_internal_basetypes_ was set to true by the client in the parameters and the client’s accepted binary format matches that of the server.
+|binary.binary_basetypes|boolean|If true, external binary format (send/recv format) may be used for some or all row field data where the field type is a built-in base type whose send/recv format is compatible with binary.binary_pg_version .
+
+May only be set if _binary.want_binary_basetypes_ was set to true by the client in the parameters and the client’s accepted send/recv format matches that of the server.
+|binary.binary_pg_version|uint16|The PostgreSQL major version that send/recv format values will be compatible with. This is not necessarily the actual upstream PostgreSQL version.
+|binary.sizeof_int|uint8|sizeof(int) on the upstream.
+|binary.sizeof_long|uint8|sizeof(long) on the upstream.
+|binary.sizeof_datum|uint8|Same as sizeof_int, but for the PostgreSQL Datum typedef.
+|binary.maxalign|uint8|Upstream PostgreSQL server’s MAXIMUM_ALIGNOF value - platform dependent, determined at build time.
+|binary.bigendian|bool|True iff the upstream is big-endian.
+|binary.float4_byval|bool|Upstream PostgreSQL’s float4_byval compile option.
+|binary.float8_byval|bool|Upstream PostgreSQL’s float8_byval compile option.
+|binary.integer_datetimes|bool|Whether TIME, TIMESTAMP and TIMESTAMP WITH TIME ZONE will be sent using integer or floating point representation.
+
+Usually this is the value of the upstream PostgreSQL’s integer_datetimes compile option.
+|===
+== Startup errors
+
+If the server rejects the client’s connection - due to non-overlapping protocol
+support, unrecognised parameter formats, unsupported required parameters like
+hooks, etc - then it will follow the startup reply message with a
++++<u>+++normal libpq protocol error message+++</u>+++. (Current versions send
+this before the startup message).
+
+== Arguments client supplies to output plugin
+
+The one opportunity for the downstream client to send information (other than replay feedback) to the upstream is at connect-time, as an array of arguments to the output plugin supplied to START LOGICAL REPLICATION.
+
+There is no back-and-forth, no handshake.
+
+As a result, the client mainly announces capabilities and makes requests of the output plugin. The output plugin will ERROR if required parameters are unset, or where incompatibilities that cannot be resolved are found. Otherwise the output plugin reports what it could and could not honour in the startup message it sends as the first message on the wire down to the client. The client chooses whether to continue replay or to disconnect and report an error to the user, then possibly reconnect with different options.
+
+=== Output plugin arguments
+
+The output plugin’s key/value arguments are specified in pairs, as key and value. They’re what’s passed to START_REPLICATION, etc.
+
+All parameters are passed in text form. They _should_ be limited to 7-bit ASCII, since the server’s text encoding is not known, but _may_ be normalized precomposed UTF-8. The types specified for parameters indicate what the output plugin should attempt to convert the text into. Clients should not send text values that are outside the range for that type.
+
+==== Capabilities
+
+Many values are capabilities flags for the client, indicating that it understands optional features like metadata caching, binary format transfers, etc. In general the output plugin _may_ disregard capabilities the client advertises as supported and act as if they are not supported. If a capability is advertised as unsupported or is not advertised the output plugin _must not_ enable the corresponding features.
+
+In other words, don’t send the client something it’s not expecting.
+
+==== Protocol versioning
+
+Two parameters max_proto_version and min_proto_version, which clients must always send, allow negotiation of the protocol version. The output plugin must ERROR if the client protocol support does not overlap its own protocol support range.
+
+The protocol version is only incremented when there are major breaking changes that all or most clients must be modified to accommodate. Most changes are done by adding new optional messages and/or by having clients advertise capabilities to opt in to features.
+
+Because these versions are expected to be incremented, to make it clear that the format of the startup parameters themselves haven’t changed, the first key/value pair _must_ be the parameter startup_params_format with value “1”.
+
+|===
+|*Key*|*Type*|*Value(s)*|*Notes*
+
+|startup_params_format|int8|1|The format version of this startup parameter set. Always the digit 1 (0x31), null terminated.
+|max_proto_version|int32|1|Newest version of the protocol supported by client. Output plugin must ERROR if supported version too old. *Required*, ERROR if missing.
+|min_proto_version|int32|1|Oldest version of the protocol supported by client. Output plugin must ERROR if supported version too old. *Required*, ERROR if missing.
+|===
+
+==== Client requirements and capabilities
+
+|===
+|*Key*|*Type*|*Default*|*Notes*
+
+|expected_encoding|string|null|The text encoding the downstream expects field values to be in. Applies to text, binary and internal representations of field values in native format. Has no effect on other protocol content. If specified, the upstream must honour it. For json protocol, must be unset or match `client_encoding`. (Current plugin versions ERROR if this is set for the native protocol and not equal to the upstream database's encoding).
+|forward_changesets|bool|false|Request that all transactions, not just those originating on the upstream, be forwarded. See “_Changeset forwarding_”.
+|want_coltypes|boolean|false|The client wants to receive data type information about columns.
+|===
+
+==== General client information
+
+These keys tell the output plugin about the client. They’re mainly for informational purposes. In particular, the versions must _not_ be used to determine compatibility for binary or send/recv format, as non-PostgreSQL clients will simply not send them at all but may still understand binary or send/recv format fields.
+
+|===
+|*Key*|*Type*|*Default*|*Notes*
+
+|pg_version_num|integer|null|PostgreSQL server_version_num of client, if it’s PostgreSQL. e.g. 090400
+|pg_version|string|null|PostgreSQL server_version of client, if it’s PostgreSQL.
+|===
+
+
+==== Parameters relating to exchange of binary values
+
+The downstream may specify to the upstream that it is capable of understanding binary (PostgreSQL internal binary datum format), and/or send/recv (PostgreSQL binary interchange) format data by setting the binary.want_binary_basetypes and/or binary.want_internal_basetypes options, or other yet-to-be-defined options.
+
+An upstream output plugin that does not support one or both formats _may_ ignore the downstream’s binary support and send text format, in which case it may ignore all binary. parameters. All downstreams _must_ support text format. An upstream output plugin _must not_ send binary or send/recv format unless the downstream has announced it can receive it. If both upstream and downstream support both formats an upstream should prefer binary format and fall back to send/recv, then to text, if compatibility requires.
+
+Internal and binary format selection should be done on a type-by-type basis. It is quite normal to send ‘text’ format for extension types while sending binary for built-in types.
+
+The downstream _must_ specify its compatibility requirements for internal and binary data if it requests either or both formats. The upstream _must_ honour these by falling back from binary to send/recv, and from send/recv to text, where the upstream and downstream are not compatible.
+
+An unspecified compatibility field _must_ presumed to be unsupported by the downstream so that older clients that don’t know about a change in a newer version don’t receive unexpected data. For example, in the unlikely event that PostgreSQL 99.8 switched to 128-bit DPD (Densely Packed Decimal) representations of NUMERIC instead of the current arbitrary-length BCD (Binary Coded Decimal) format, a new binary.dpd_numerics parameter would be added. Clients that didn’t know about the change wouldn’t know to set it, so the upstream would presume it unsupported and send text format NUMERIC to those clients. This also means that clients that support the new format wouldn’t be able to receive the old format in binary from older servers since they’d specify dpd_numerics = true in their compatibility parameters.
+
+At this time a downstream may specify compatibility with only one value for a given option; i.e. a downstream cannot say it supports both 4-byte and 8-byte sizeof(int). Leaving it unspecified means the upstream must assume the downstream supports neither. (A future protocol extension may allow clients to specify alternative sets of supported formats).
+
+The `pg_version` option _must not_ be used to decide compatibility. Use `binary.basetypes_major_version` instead.
+
+|===
+|*Key name*|*Value type*|*Default*|*Description*
+
+|binary.want_binary_basetypes|boolean|false|True if the client accepts binary interchange (send/recv) format rows for PostgreSQL built-in base types.
+|binary.want_internal_basetypes|boolean|false|True if the client accepts PostgreSQL internal-format binary output for base PostgreSQL types not otherwise specified elsewhere.
+|binary.basetypes_major_version|uint16|null|The PostgreSQL major version (x.y) the downstream expects binary and send/recv format values to be in. Represented as an integer in XXYY format (no leading zero since it’s an integer), e.g. 9.5 is 905. This corresponds to PG_VERSION_NUM/100 in PostgreSQL.
+|binary.sizeof_int|uint8|+null+|sizeof(int) on the downstream.
+|binary.sizeof_long|uint8|null|sizeof(long) on the downstream.
+|binary.sizeof_datum|uint8|null|Same as sizeof_int, but for the PostgreSQL Datum typedef.
+|binary.maxalign|uint8|null|Downstream PostgreSQL server’s maxalign value - platform dependent, determined at build time.
+|binary.bigendian|bool|null|True iff the downstream is big-endian.
+|binary.float4_byval|bool|null|Downstream PostgreSQL’s float4_byval compile option.
+|binary.float8_byval|bool|null|Downstream PostgreSQL’s float8_byval compile option.
+|binary.integer_datetimes|bool|null|Downstream PostgreSQL’s integer_datetimes compile option.
+|===
+
+== Extensibility
+
+Because of the use of optional parameters in output plugin arguments, and the
+confirmation/response sent in the startup packet, a basic handshake is possible
+between upstream and downstream, allowing negotiation of capabilities.
+
+The output plugin must never send non-optional data or change its wire format
+without confirmation from the client that it can understand the new data. It
+may send optional data without negotiation.
+
+When extending the output plugin arguments, add-ons are expected to prefix all
+keys with the extension name, and should preferably use a single top level key
+with a json object value to carry their extension information. Additions to the
+startup message should follow the same pattern.
+
+Hooks and plugins can be used to add functionality specific to a client.
+
+== JSON protocol
+
+If `proto_format` is set to `json` then the output plugin will emit JSON
+instead of the custom binary protocol. JSON support is intended mainly for
+debugging and diagnostics.
+
+The JSON format supports all the same hooks.
diff --git a/contrib/pglogical_output/expected/basic_json.out b/contrib/pglogical_output/expected/basic_json.out
new file mode 100644
index 0000000..b57577b
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_json.out
@@ -0,0 +1,108 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ data
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action":"S", "params": {"max_proto_version":"1","min_proto_version":"1","coltypes":"f","pg_version_num":"90600","pg_version":"9.6devel","pg_catversion":"201510161","database_encoding":"UTF8","encoding":"UTF8","forward_changesets":"f","forward_changeset_origins":"f","binary.internal_basetypes":"f","binary.binary_basetypes":"f","binary.basetypes_major_version":"906","binary.sizeof_int":"4","binary.sizeof_long":"8","binary.sizeof_datum":"8","binary.maxalign":"8","binary.bigendian":"f","binary.float4_byval":"t","binary.float8_byval":"t","binary.integer_datetimes":"t","binary.binary_pg_version":"906","no_txinfo":"t","hooks.startup_hook_enabled":"f","hooks.shutdown_hook_enabled":"f","hooks.row_filter_enabled":"f","hooks.transaction_filter_enabled":"f"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":1,"tx":"textval","ts":null,"jsb":null,"js":null,"ba":null}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":2,"tx":null,"ts":null,"jsb":null,"js":null,"ba":"\\xdeadbeef0001"}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":3,"tx":"blah","ts":"2045-09-12T12:34:56","jsb":null,"js":null,"ba":null}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":4,"tx":null,"ts":null,"jsb":{"key": "value"},"js":{"key":"value"},"ba":null}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":6,"tx":"row1","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":7,"tx":"row2","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":8,"tx":"row3","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"D","relation":["public","demo"],"oldtuple":{"seq":7,"tx":null,"ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"U","relation":["public","demo"],"newtuple":{"seq":6,"tx":"updated","ts":null,"jsb":null,"js":null,"ba":null}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"t"}
+ {"action":"I","relation":["public","cat_test"],"newtuple":{"id":42}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":9,"tx":"1","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":10,"tx":"2","ts":null,"jsb":null,"js":null,"ba":null}}
+ {{"action":"C"}}
+(27 rows)
+
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/basic_native.out b/contrib/pglogical_output/expected/basic_native.out
new file mode 100644
index 0000000..a7c88f3
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_native.out
@@ -0,0 +1,99 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ count
+-------
+ 39
+(1 row)
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+ count
+-------
+ 39
+(1 row)
+
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/encoding_json.out b/contrib/pglogical_output/expected/encoding_json.out
new file mode 100644
index 0000000..82c719a
--- /dev/null
+++ b/contrib/pglogical_output/expected/encoding_json.out
@@ -0,0 +1,59 @@
+SET synchronous_commit = on;
+-- This file doesn't share common setup with the native tests,
+-- since it's specific to how the text protocol handles encodings.
+CREATE TABLE enctest(blah text);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+SET client_encoding = 'UTF-8';
+INSERT INTO enctest(blah)
+VALUES
+('áàä'),('fl'), ('½⅓'), ('カンジ');
+RESET client_encoding;
+SET client_encoding = 'LATIN-1';
+-- Will ERROR, explicit encoding request doesn't match client_encoding
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: expected_encoding must be unset or match client_encoding in text protocols
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Will succeed since we don't request any encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: character with byte sequence 0xef 0xac 0x82 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+-- Will succeed since it matches the current encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'LATIN-1',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: character with byte sequence 0xef 0xac 0x82 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+RESET client_encoding;
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE enctest;
diff --git a/contrib/pglogical_output/expected/hooks_json.out b/contrib/pglogical_output/expected/hooks_json.out
new file mode 100644
index 0000000..be20ed8
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_json.out
@@ -0,0 +1,137 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Test table filter
+SELECT data FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ data
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action":"S", "params": {"max_proto_version":"1","min_proto_version":"1","coltypes":"f","pg_version_num":"90600","pg_version":"9.6devel","pg_catversion":"201510161","database_encoding":"UTF8","encoding":"UTF8","forward_changesets":"f","forward_changeset_origins":"f","binary.internal_basetypes":"f","binary.binary_basetypes":"f","binary.basetypes_major_version":"906","binary.sizeof_int":"4","binary.sizeof_long":"8","binary.sizeof_datum":"8","binary.maxalign":"8","binary.bigendian":"f","binary.float4_byval":"t","binary.float8_byval":"t","binary.integer_datetimes":"t","binary.binary_pg_version":"906","no_txinfo":"t","hooks.startup_hook_enabled":"t","hooks.shutdown_hook_enabled":"t","hooks.row_filter_enabled":"t","hooks.transaction_filter_enabled":"t"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":1}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":2}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":3}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":4}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":5}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":6}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":7}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":8}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":9}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":10}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"U","relation":["public","test_nofilt"],"newtuple":{"id":500}}
+ {{"action":"C"}}
+(24 rows)
+
+-- test action filter
+SELECT data FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ data
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action":"S", "params": {"max_proto_version":"1","min_proto_version":"1","coltypes":"f","pg_version_num":"90600","pg_version":"9.6devel","pg_catversion":"201510161","database_encoding":"UTF8","encoding":"UTF8","forward_changesets":"f","forward_changeset_origins":"f","binary.internal_basetypes":"f","binary.binary_basetypes":"f","binary.basetypes_major_version":"906","binary.sizeof_int":"4","binary.sizeof_long":"8","binary.sizeof_datum":"8","binary.maxalign":"8","binary.bigendian":"f","binary.float4_byval":"t","binary.float8_byval":"t","binary.integer_datetimes":"t","binary.binary_pg_version":"906","no_txinfo":"t","hooks.startup_hook_enabled":"t","hooks.shutdown_hook_enabled":"t","hooks.row_filter_enabled":"t","hooks.transaction_filter_enabled":"t"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":1}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":2}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":3}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":4}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":5}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":6}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":7}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":8}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":9}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":10}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":1}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":2}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":3}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":4}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":5}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":6}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":7}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":8}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":9}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":10}}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+ {"action":"B", has_catalog_changes:"f"}
+ {{"action":"C"}}
+(33 rows)
+
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/hooks_native.out b/contrib/pglogical_output/expected/hooks_native.out
new file mode 100644
index 0000000..4a547cb
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_native.out
@@ -0,0 +1,104 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Regular hook setup
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+ count
+-------
+ 40
+(1 row)
+
+-- Test action filter
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+ count
+-------
+ 53
+(1 row)
+
+-- Invalid row fiter hook function
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+ERROR: function public.nosuchfunction(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Hook filter functoin with wrong signature
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+ERROR: function public.wrong_signature_fn(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/params_native.out b/contrib/pglogical_output/expected/params_native.out
new file mode 100644
index 0000000..9475035
--- /dev/null
+++ b/contrib/pglogical_output/expected/params_native.out
@@ -0,0 +1,118 @@
+SET synchronous_commit = on;
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+--
+-- Various invalid parameter combos:
+--
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: logical decoding output plugin "pglogical_output" produces binary output, but function "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+ERROR: client sent startup parameters in format 2 but we only support format 1
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: unrecognised encoding name bork passed to expected_encoding
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+ERROR: client requested protocol invalid but only "json" or "native" are supported
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+ data
+------
+(0 rows)
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
diff --git a/contrib/pglogical_output/expected/params_native_1.out b/contrib/pglogical_output/expected/params_native_1.out
new file mode 100644
index 0000000..5607d50
--- /dev/null
+++ b/contrib/pglogical_output/expected/params_native_1.out
@@ -0,0 +1,94 @@
+SET synchronous_commit = on;
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+--
+-- Various invalid parameter combos:
+--
+-- Text mode is not supported
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: logical decoding output plugin "pglogical_output" produces binary output, but function "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+ERROR: client sent startup parameters in format 2 but we only support format 1
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: only "UTF8" encoding is supported by this server, client requested bork
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Currently we're sensitive to the encoding name's format (TODO)
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: only "UTF8" encoding is supported by this server, client requested UTF-8
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
diff --git a/contrib/pglogical_output/expected/pre_clean.out b/contrib/pglogical_output/expected/pre_clean.out
new file mode 100644
index 0000000..5c12dc0
--- /dev/null
+++ b/contrib/pglogical_output/expected/pre_clean.out
@@ -0,0 +1,10 @@
+DO
+LANGUAGE plpgsql
+$$
+BEGIN
+ IF EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'regression_slot')
+ THEN
+ PERFORM pg_drop_replication_slot('regression_slot');
+ END IF;
+END;
+$$;
diff --git a/contrib/pglogical_output/pglogical_config.c b/contrib/pglogical_output/pglogical_config.c
new file mode 100644
index 0000000..cc22700
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.c
@@ -0,0 +1,499 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_config.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_config.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+
+#include "catalog/catversion.h"
+#include "catalog/namespace.h"
+
+#include "mb/pg_wchar.h"
+
+#include "nodes/makefuncs.h"
+
+#include "utils/builtins.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+typedef enum PGLogicalOutputParamType
+{
+ OUTPUT_PARAM_TYPE_BOOL,
+ OUTPUT_PARAM_TYPE_UINT32,
+ OUTPUT_PARAM_TYPE_STRING,
+ OUTPUT_PARAM_TYPE_QUALIFIED_NAME
+} PGLogicalOutputParamType;
+
+/* param parsing */
+static Datum get_param_value(DefElem *elem, bool null_ok,
+ PGLogicalOutputParamType type);
+
+static Datum get_param(List *options, const char *name, bool missing_ok,
+ bool null_ok, PGLogicalOutputParamType type,
+ bool *found);
+static bool parse_param_bool(DefElem *elem);
+static uint32 parse_param_uint32(DefElem *elem);
+
+static void
+process_parameters_v1(List *options, PGLogicalOutputData *data);
+
+enum {
+ PARAM_UNRECOGNISED,
+ PARAM_MAX_PROTOCOL_VERSION,
+ PARAM_MIN_PROTOCOL_VERSION,
+ PARAM_PROTOCOL_FORMAT,
+ PARAM_EXPECTED_ENCODING,
+ PARAM_BINARY_BIGENDIAN,
+ PARAM_BINARY_SIZEOF_DATUM,
+ PARAM_BINARY_SIZEOF_INT,
+ PARAM_BINARY_SIZEOF_LONG,
+ PARAM_BINARY_FLOAT4BYVAL,
+ PARAM_BINARY_FLOAT8BYVAL,
+ PARAM_BINARY_INTEGER_DATETIMES,
+ PARAM_BINARY_WANT_INTERNAL_BASETYPES,
+ PARAM_BINARY_WANT_BINARY_BASETYPES,
+ PARAM_BINARY_BASETYPES_MAJOR_VERSION,
+ PARAM_PG_VERSION,
+ PARAM_FORWARD_CHANGESETS,
+ PARAM_HOOKS_SETUP_FUNCTION,
+ PARAM_NO_TXINFO
+} OutputPluginParamKey;
+
+typedef struct {
+ const char * const paramname;
+ int paramkey;
+} OutputPluginParam;
+
+/* Oh, if only C had switch on strings */
+static OutputPluginParam param_lookup[] = {
+ {"max_proto_version", PARAM_MAX_PROTOCOL_VERSION},
+ {"min_proto_version", PARAM_MIN_PROTOCOL_VERSION},
+ {"proto_format", PARAM_PROTOCOL_FORMAT},
+ {"expected_encoding", PARAM_EXPECTED_ENCODING},
+ {"binary.bigendian", PARAM_BINARY_BIGENDIAN},
+ {"binary.sizeof_datum", PARAM_BINARY_SIZEOF_DATUM},
+ {"binary.sizeof_int", PARAM_BINARY_SIZEOF_INT},
+ {"binary.sizeof_long", PARAM_BINARY_SIZEOF_LONG},
+ {"binary.float4_byval", PARAM_BINARY_FLOAT4BYVAL},
+ {"binary.float8_byval", PARAM_BINARY_FLOAT8BYVAL},
+ {"binary.integer_datetimes", PARAM_BINARY_INTEGER_DATETIMES},
+ {"binary.want_internal_basetypes", PARAM_BINARY_WANT_INTERNAL_BASETYPES},
+ {"binary.want_binary_basetypes", PARAM_BINARY_WANT_BINARY_BASETYPES},
+ {"binary.basetypes_major_version", PARAM_BINARY_BASETYPES_MAJOR_VERSION},
+ {"pg_version", PARAM_PG_VERSION},
+ {"forward_changesets", PARAM_FORWARD_CHANGESETS},
+ {"hooks.setup_function", PARAM_HOOKS_SETUP_FUNCTION},
+ {"no_txinfo", PARAM_NO_TXINFO},
+ {NULL, PARAM_UNRECOGNISED}
+};
+
+/*
+ * Look up a param name to find the enum value for the
+ * param, or PARAM_UNRECOGNISED if not found.
+ */
+static int
+get_param_key(const char * const param_name)
+{
+ OutputPluginParam *param = ¶m_lookup[0];
+
+ do {
+ if (strcmp(param->paramname, param_name) == 0)
+ return param->paramkey;
+ param++;
+ } while (param->paramname != NULL);
+
+ return PARAM_UNRECOGNISED;
+}
+
+
+void
+process_parameters_v1(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ ListCell *lc;
+
+ /*
+ * max_proto_version and min_proto_version are specified
+ * as required, and must be parsed before anything else.
+ *
+ * TODO: We should still parse them as optional and
+ * delay the ERROR until after the startup reply.
+ */
+ val = get_param(options, "max_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_max_proto_version = DatumGetUInt32(val);
+
+ val = get_param(options, "min_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_min_proto_version = DatumGetUInt32(val);
+
+ /* Examine all the other params in the v1 message. */
+ foreach(lc, options)
+ {
+ DefElem *elem = lfirst(lc);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Check each param, whether or not we recognise it */
+ switch(get_param_key(elem->defname))
+ {
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+
+ case PARAM_BINARY_BIGENDIAN:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_bigendian_set = true;
+ data->client_binary_bigendian = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_DATUM:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofdatum = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_INT:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofint = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_LONG:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeoflong = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_FLOAT4BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_FLOAT8BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_INTEGER_DATETIMES:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_intdatetimes_set = true;
+ data->client_binary_intdatetimes = DatumGetBool(val);
+ break;
+
+ case PARAM_PROTOCOL_FORMAT:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_STRING);
+ data->client_protocol_format = DatumGetCString(val);
+ break;
+
+ case PARAM_EXPECTED_ENCODING:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_STRING);
+ data->client_expected_encoding = DatumGetCString(val);
+ break;
+
+ case PARAM_PG_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_pg_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_FORWARD_CHANGESETS:
+ /*
+ * Check to see if the client asked for changeset forwarding
+ *
+ * Note that we cannot support this on 9.4. We'll tell the client
+ * in the startup reply message.
+ */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_forward_changesets_set = true;
+ data->client_forward_changesets = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_WANT_INTERNAL_BASETYPES:
+ /* check if we want to use internal data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_internal_basetypes_set = true;
+ data->client_want_internal_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_WANT_BINARY_BASETYPES:
+ /* check if we want to use binary data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_binary_basetypes_set = true;
+ data->client_want_binary_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_BASETYPES_MAJOR_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_basetypes_major_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_HOOKS_SETUP_FUNCTION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_QUALIFIED_NAME);
+ data->hooks_setup_funcname = (List*) PointerGetDatum(val);
+ break;
+
+ case PARAM_NO_TXINFO:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_no_txinfo = DatumGetBool(val);
+ break;
+
+ case PARAM_UNRECOGNISED:
+ ereport(DEBUG1,
+ (errmsg("Unrecognised pglogical parameter %s ignored", elem->defname)));
+ break;
+ }
+ }
+}
+
+/*
+ * Read parameters sent by client at startup and store recognised
+ * ones in the parameters PGLogicalOutputData.
+ *
+ * The PGLogicalOutputData must have all client-surprised parameter fields
+ * zeroed, such as by memset or palloc0, since values not supplied
+ * by the client are not set.
+ */
+int
+process_parameters(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ int params_format;
+
+ val = get_param(options, "startup_params_format", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+
+ params_format = DatumGetUInt32(val);
+
+ if (params_format == 1)
+ {
+ process_parameters_v1(options, data);
+ }
+
+ return params_format;
+}
+
+static Datum
+get_param_value(DefElem *elem, bool null_ok, PGLogicalOutputParamType type)
+{
+ /* Check for NULL value */
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ {
+ if (null_ok)
+ return (Datum) 0;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("parameter \"%s\" cannot be NULL", elem->defname)));
+ }
+
+ switch (type)
+ {
+ case OUTPUT_PARAM_TYPE_UINT32:
+ return UInt32GetDatum(parse_param_uint32(elem));
+ case OUTPUT_PARAM_TYPE_BOOL:
+ return BoolGetDatum(parse_param_bool(elem));
+ case OUTPUT_PARAM_TYPE_STRING:
+ return PointerGetDatum(pstrdup(strVal(elem->arg)));
+ case OUTPUT_PARAM_TYPE_QUALIFIED_NAME:
+ return PointerGetDatum(textToQualifiedNameList(cstring_to_text(pstrdup(strVal(elem->arg)))));
+ default:
+ elog(ERROR, "unknown parameter type %d", type);
+ }
+}
+
+/*
+ * Param parsing
+ *
+ * This is not exactly fast but since it's only called on replication start
+ * we'll leave it for now.
+ */
+static Datum
+get_param(List *options, const char *name, bool missing_ok, bool null_ok,
+ PGLogicalOutputParamType type, bool *found)
+{
+ ListCell *option;
+
+ *found = false;
+
+ foreach(option, options)
+ {
+ DefElem *elem = lfirst(option);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Search until matching parameter found */
+ if (pg_strcasecmp(name, elem->defname))
+ continue;
+
+ *found = true;
+
+ return get_param_value(elem, null_ok, type);
+ }
+
+ if (!missing_ok)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("missing required parameter \"%s\"", name)));
+
+ return (Datum) 0;
+}
+
+static bool
+parse_param_bool(DefElem *elem)
+{
+ bool res;
+
+ if (!parse_bool(strVal(elem->arg), &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse boolean value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return res;
+}
+
+static uint32
+parse_param_uint32(DefElem *elem)
+{
+ int64 res;
+
+ if (!scanint8(strVal(elem->arg), true, &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse integer value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ if (res > PG_UINT32_MAX || res < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("value \"%s\" out of range for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return (uint32) res;
+}
+
+static List*
+add_startup_msg_s(List *l, char *key, char *val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(val)));
+}
+
+static List*
+add_startup_msg_i(List *l, char *key, int val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(psprintf("%d", val))));
+}
+
+static List*
+add_startup_msg_b(List *l, char *key, bool val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(val ? "t" : "f")));
+}
+
+/*
+ * This builds the protocol startup message, which is always the first
+ * message on the wire after the client sends START_REPLICATION.
+ *
+ * It confirms to the client that we could apply requested options, and
+ * tells the client our capabilities.
+ *
+ * Any additional parameters provided by the startup hook are also output
+ * now.
+ *
+ * The output param 'msg' is a null-terminated char* palloc'd in the current
+ * memory context and the length 'len' of that string that is valid. The caller
+ * should pfree the result after use.
+ *
+ * This is a bit less efficient than direct pq_sendblah calls, but
+ * separates config handling from the protocol implementation, and
+ * it's not like startup msg performance matters much.
+ */
+List *
+prepare_startup_message(PGLogicalOutputData *data)
+{
+ ListCell *lc;
+ List *l = NIL;
+
+ l = add_startup_msg_s(l, "max_proto_version", "1");
+ l = add_startup_msg_s(l, "min_proto_version", "1");
+
+ /* We don't support understand column types yet */
+ l = add_startup_msg_b(l, "coltypes", false);
+
+ /* Info about our Pg host */
+ l = add_startup_msg_i(l, "pg_version_num", PG_VERSION_NUM);
+ l = add_startup_msg_s(l, "pg_version", PG_VERSION);
+ l = add_startup_msg_i(l, "pg_catversion", CATALOG_VERSION_NO);
+
+ l = add_startup_msg_s(l, "database_encoding", (char*)GetDatabaseEncodingName());
+
+ l = add_startup_msg_s(l, "encoding", (char*)pg_encoding_to_char(data->field_datum_encoding));
+
+ l = add_startup_msg_b(l, "forward_changesets",
+ data->forward_changesets);
+ l = add_startup_msg_b(l, "forward_changeset_origins",
+ data->forward_changeset_origins);
+
+ /* binary options enabled */
+ l = add_startup_msg_b(l, "binary.internal_basetypes",
+ data->allow_internal_basetypes);
+ l = add_startup_msg_b(l, "binary.binary_basetypes",
+ data->allow_binary_basetypes);
+
+ /* Binary format characteristics of server */
+ l = add_startup_msg_i(l, "binary.basetypes_major_version", PG_VERSION_NUM/100);
+ l = add_startup_msg_i(l, "binary.sizeof_int", sizeof(int));
+ l = add_startup_msg_i(l, "binary.sizeof_long", sizeof(long));
+ l = add_startup_msg_i(l, "binary.sizeof_datum", sizeof(Datum));
+ l = add_startup_msg_i(l, "binary.maxalign", MAXIMUM_ALIGNOF);
+ l = add_startup_msg_b(l, "binary.bigendian", server_bigendian());
+ l = add_startup_msg_b(l, "binary.float4_byval", server_float4_byval());
+ l = add_startup_msg_b(l, "binary.float8_byval", server_float8_byval());
+ l = add_startup_msg_b(l, "binary.integer_datetimes", server_integer_datetimes());
+ /* We don't know how to send in anything except our host's format */
+ l = add_startup_msg_i(l, "binary.binary_pg_version",
+ PG_VERSION_NUM/100);
+
+ l = add_startup_msg_b(l, "no_txinfo", data->client_no_txinfo);
+
+
+ /*
+ * Confirm that we've enabled any requested hook functions.
+ */
+ l = add_startup_msg_b(l, "hooks.startup_hook_enabled",
+ data->hooks.startup_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.shutdown_hook_enabled",
+ data->hooks.shutdown_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.row_filter_enabled",
+ data->hooks.row_filter_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.transaction_filter_enabled",
+ data->hooks.txn_filter_hook != NULL);
+
+ /*
+ * Output any extra params supplied by a startup hook by appending
+ * them verbatim to the params list.
+ */
+ foreach(lc, data->extra_startup_params)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ l = lappend(l, param);
+ }
+
+ return l;
+}
diff --git a/contrib/pglogical_output/pglogical_config.h b/contrib/pglogical_output/pglogical_config.h
new file mode 100644
index 0000000..3af3ce8
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.h
@@ -0,0 +1,55 @@
+#ifndef PG_LOGICAL_CONFIG_H
+#define PG_LOGICAL_CONFIG_H
+
+#ifndef PG_VERSION_NUM
+#error <postgres.h> must be included first
+#endif
+
+inline static bool
+server_float4_byval(void)
+{
+#ifdef USE_FLOAT4_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_float8_byval(void)
+{
+#ifdef USE_FLOAT8_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_integer_datetimes(void)
+{
+#ifdef USE_INTEGER_DATETIMES
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_bigendian(void)
+{
+#ifdef WORDS_BIGENDIAN
+ return true;
+#else
+ return false;
+#endif
+}
+
+typedef struct List List;
+typedef struct PGLogicalOutputData PGLogicalOutputData;
+
+extern int process_parameters(List *options, PGLogicalOutputData *data);
+
+extern List * prepare_startup_message(PGLogicalOutputData *data);
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_hooks.c b/contrib/pglogical_output/pglogical_hooks.c
new file mode 100644
index 0000000..73e8120
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.c
@@ -0,0 +1,232 @@
+#include "postgres.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#include "replication/origin.h"
+
+#include "parser/parse_func.h"
+
+#include "utils/acl.h"
+#include "utils/lsyscache.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_hooks.h"
+#include "pglogical_output.h"
+
+/*
+ * Returns Oid of the hooks function specified in funcname.
+ *
+ * Error is thrown if function doesn't exist or doen't return correct datatype
+ * or is volatile.
+ */
+static Oid
+get_hooks_function_oid(List *funcname)
+{
+ Oid funcid;
+ Oid funcargtypes[1];
+
+ funcargtypes[0] = INTERNALOID;
+
+ /* find the the function */
+ funcid = LookupFuncName(funcname, 1, funcargtypes, false);
+
+ /* Validate that the function returns void */
+ if (get_func_rettype(funcid) != VOIDOID)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must return void",
+ NameListToString(funcname))));
+ }
+
+ if (func_volatile(funcid) == PROVOLATILE_VOLATILE)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must not be VOLATILE",
+ NameListToString(funcname))));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(funcname))));
+ }
+
+ return funcid;
+}
+
+/*
+ * If a hook setup function was specified in the startup parameters, look it up
+ * in the catalogs, check permissions, call it, and store the resulting hook
+ * info struct.
+ */
+void
+load_hooks(PGLogicalOutputData *data)
+{
+ Oid hooks_func;
+ MemoryContext old_ctxt;
+ bool txn_started = false;
+
+ if (!IsTransactionState())
+ {
+ txn_started = true;
+ StartTransactionCommand();
+ }
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+ hooks_func = get_hooks_function_oid(data->hooks_setup_funcname);
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) OidFunctionCall1(hooks_func, PointerGetDatum(&data->hooks));
+ MemoryContextSwitchTo(old_ctxt);
+
+ elog(DEBUG3, "pglogical_output: Loaded hooks from function %u. Hooks are: \n"
+ "\tstartup_hook: %p\n"
+ "\tshutdown_hook: %p\n"
+ "\trow_filter_hook: %p\n"
+ "\ttxn_filter_hook: %p\n"
+ "\thooks_private_data: %p\n",
+ hooks_func,
+ data->hooks.startup_hook,
+ data->hooks.shutdown_hook,
+ data->hooks.row_filter_hook,
+ data->hooks.txn_filter_hook,
+ data->hooks.hooks_private_data);
+ }
+
+ if (txn_started)
+ CommitTransactionCommand();
+}
+
+void
+call_startup_hook(PGLogicalOutputData *data, List *plugin_params)
+{
+ struct PGLogicalStartupHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.startup_hook != NULL)
+ {
+ bool tx_started = false;
+
+ args.private_data = data->hooks.hooks_private_data;
+ args.in_params = plugin_params;
+ args.out_params = NIL;
+
+ elog(DEBUG3, "calling pglogical startup hook");
+
+ if (!IsTransactionState())
+ {
+ tx_started = true;
+ StartTransactionCommand();
+ }
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.startup_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ if (tx_started)
+ CommitTransactionCommand();
+
+ data->extra_startup_params = args.out_params;
+ /* The startup hook might change the private data seg */
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical startup hook");
+ }
+}
+
+void
+call_shutdown_hook(PGLogicalOutputData *data)
+{
+ struct PGLogicalShutdownHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.shutdown_hook != NULL)
+ {
+ args.private_data = data->hooks.hooks_private_data;
+
+ elog(DEBUG3, "calling pglogical shutdown hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.shutdown_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical shutdown hook");
+ }
+}
+
+/*
+ * Decide if the individual change should be filtered out by
+ * calling a client-provided hook.
+ */
+bool
+call_row_filter_hook(PGLogicalOutputData *data, ReorderBufferTXN *txn,
+ Relation rel, ReorderBufferChange *change)
+{
+ struct PGLogicalRowFilterArgs hook_args;
+ MemoryContext old_ctxt;
+ bool ret = true;
+
+ if (data->hooks.row_filter_hook != NULL)
+ {
+ hook_args.change_type = change->action;
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.changed_rel = rel;
+
+ elog(DEBUG3, "calling pglogical row filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.row_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical row filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
+
+bool
+call_txn_filter_hook(PGLogicalOutputData *data, RepOriginId txn_origin)
+{
+ struct PGLogicalTxnFilterArgs hook_args;
+ bool ret = true;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.txn_filter_hook != NULL)
+ {
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.origin_id = txn_origin;
+
+ elog(DEBUG3, "calling pglogical txn filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.txn_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical txn filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
diff --git a/contrib/pglogical_output/pglogical_hooks.h b/contrib/pglogical_output/pglogical_hooks.h
new file mode 100644
index 0000000..df661f3
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.h
@@ -0,0 +1,22 @@
+#ifndef PGLOGICAL_HOOKS_H
+#define PGLOGICAL_HOOKS_H
+
+#include "replication/reorderbuffer.h"
+
+/* public interface for hooks */
+#include "pglogical_output/hooks.h"
+
+extern void load_hooks(PGLogicalOutputData *data);
+
+extern void call_startup_hook(PGLogicalOutputData *data, List *plugin_params);
+
+extern void call_shutdown_hook(PGLogicalOutputData *data);
+
+extern bool call_row_filter_hook(PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, Relation rel, ReorderBufferChange *change);
+
+extern bool call_txn_filter_hook(PGLogicalOutputData *data,
+ RepOriginId txn_origin);
+
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_output.c b/contrib/pglogical_output/pglogical_output.c
new file mode 100644
index 0000000..b8fc55e
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.c
@@ -0,0 +1,537 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+#include "pglogical_hooks.h"
+
+#include "access/hash.h"
+#include "access/sysattr.h"
+#include "access/xact.h"
+
+#include "catalog/pg_class.h"
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#include "mb/pg_wchar.h"
+
+#include "nodes/parsenodes.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/output_plugin.h"
+#include "replication/logical.h"
+#include "replication/origin.h"
+
+#include "utils/builtins.h"
+#include "utils/catcache.h"
+#include "utils/guc.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+PG_MODULE_MAGIC;
+
+extern void _PG_output_plugin_init(OutputPluginCallbacks *cb);
+
+/* These must be available to pg_dlsym() */
+static void pg_decode_startup(LogicalDecodingContext * ctx,
+ OutputPluginOptions *opt, bool is_init);
+static void pg_decode_shutdown(LogicalDecodingContext * ctx);
+static void pg_decode_begin_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn);
+static void pg_decode_commit_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+static void pg_decode_change(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, Relation rel,
+ ReorderBufferChange *change);
+
+static bool pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id);
+
+static void send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message);
+
+static bool startup_message_sent = false;
+
+/* specify output plugin callbacks */
+void
+_PG_output_plugin_init(OutputPluginCallbacks *cb)
+{
+ AssertVariableIsOfType(&_PG_output_plugin_init, LogicalOutputPluginInit);
+
+ cb->startup_cb = pg_decode_startup;
+ cb->begin_cb = pg_decode_begin_txn;
+ cb->change_cb = pg_decode_change;
+ cb->commit_cb = pg_decode_commit_txn;
+ cb->filter_by_origin_cb = pg_decode_origin_filter;
+ cb->shutdown_cb = pg_decode_shutdown;
+}
+
+static bool
+check_binary_compatibility(PGLogicalOutputData *data)
+{
+ if (data->client_binary_basetypes_major_version != PG_VERSION_NUM / 100)
+ return false;
+
+ if (data->client_binary_bigendian_set
+ && data->client_binary_bigendian != server_bigendian())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian mis-match");
+ return false;
+ }
+
+ if (data->client_binary_sizeofdatum != 0
+ && data->client_binary_sizeofdatum != sizeof(Datum))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(Datum) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeofint != 0
+ && data->client_binary_sizeofint != sizeof(int))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(int) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeoflong != 0
+ && data->client_binary_sizeoflong != sizeof(long))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(long) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float4byval_set
+ && data->client_binary_float4byval != server_float4_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float4byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float8byval_set
+ && data->client_binary_float8byval != server_float8_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float8byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_intdatetimes_set
+ && data->client_binary_intdatetimes != server_integer_datetimes())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian integer datetimes mismatch");
+ return false;
+ }
+
+ return true;
+}
+
+/* initialize this plugin */
+static void
+pg_decode_startup(LogicalDecodingContext * ctx, OutputPluginOptions *opt,
+ bool is_init)
+{
+ PGLogicalOutputData *data = palloc0(sizeof(PGLogicalOutputData));
+
+ data->context = AllocSetContextCreate(TopMemoryContext,
+ "pglogical conversion context",
+ ALLOCSET_DEFAULT_MINSIZE,
+ ALLOCSET_DEFAULT_INITSIZE,
+ ALLOCSET_DEFAULT_MAXSIZE);
+ data->allow_internal_basetypes = false;
+ data->allow_binary_basetypes = false;
+
+
+ ctx->output_plugin_private = data;
+
+ /*
+ * This is replication start and not slot initialization.
+ *
+ * Parse and validate options passed by the client.
+ */
+ if (!is_init)
+ {
+ int params_format;
+
+ /*
+ * Ideally we'd send the startup message immediately. That way
+ * it'd arrive before any error we emit if we see incompatible
+ * options sent by the client here. That way the client could
+ * possibly adjust its options and reconnect. It'd also make
+ * sure the client gets the startup message in a timely way if
+ * the server is idle, since otherwise it could be a while
+ * before the next callback.
+ *
+ * The decoding plugin API doesn't let us write to the stream
+ * from here, though, so we have to delay the startup message
+ * until the first change processed on the stream, in a begin
+ * callback.
+ *
+ * If we ERROR there, the startup message is buffered but not
+ * sent since the callback didn't finish. So we'd have to send
+ * the startup message, finish the callback and check in the
+ * next callback if we need to ERROR.
+ *
+ * That's a bit much hoop jumping, so for now ERRORs are
+ * immediate. A way to emit a message from the startup callback
+ * is really needed to change that.
+ */
+ startup_message_sent = false;
+
+ /* Now parse the rest of the params and ERROR if we see any we don't recognise */
+ params_format = process_parameters(ctx->output_plugin_options, data);
+
+ if (params_format != 1)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent startup parameters in format %d but we only support format 1",
+ params_format)));
+
+ if (data->client_min_proto_version > PG_LOGICAL_PROTO_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent min_proto_version=%d but we only support protocol %d or lower",
+ data->client_min_proto_version, PG_LOGICAL_PROTO_VERSION_NUM)));
+
+ if (data->client_max_proto_version < PG_LOGICAL_PROTO_MIN_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent max_proto_version=%d but we only support protocol %d or higher",
+ data->client_max_proto_version, PG_LOGICAL_PROTO_MIN_VERSION_NUM)));
+
+ /*
+ * Set correct protocol format.
+ *
+ * This is the output plugin protocol format, this is different
+ * from the individual fields binary vs textual format.
+ */
+ if (data->client_protocol_format != NULL
+ && strcmp(data->client_protocol_format, "json") == 0)
+ {
+ data->api = pglogical_init_api(PGLogicalProtoJson);
+ opt->output_type = OUTPUT_PLUGIN_TEXTUAL_OUTPUT;
+ }
+ else if ((data->client_protocol_format != NULL
+ && strcmp(data->client_protocol_format, "native") == 0)
+ || data->client_protocol_format == NULL)
+ {
+ data->api = pglogical_init_api(PGLogicalProtoNative);
+ opt->output_type = OUTPUT_PLUGIN_BINARY_OUTPUT;
+
+ if (data->client_no_txinfo)
+ {
+ elog(WARNING, "no_txinfo option ignored for protocols other than json");
+ data->client_no_txinfo = false;
+ }
+ }
+ else
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client requested protocol %s but only \"json\" or \"native\" are supported",
+ data->client_protocol_format)));
+ }
+
+ /* check for encoding match if specific encoding demanded by client */
+ if (data->client_expected_encoding != NULL
+ && strlen(data->client_expected_encoding) != 0)
+ {
+ int wanted_encoding = pg_char_to_encoding(data->client_expected_encoding);
+
+ if (wanted_encoding == -1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("unrecognised encoding name %s passed to expected_encoding",
+ data->client_expected_encoding)));
+
+ if (opt->output_type == OUTPUT_PLUGIN_TEXTUAL_OUTPUT)
+ {
+ /*
+ * datum encoding must match assigned client_encoding in text
+ * proto, since everything is subject to client_encoding
+ * conversion.
+ */
+ if (wanted_encoding != pg_get_client_encoding())
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("expected_encoding must be unset or match client_encoding in text protocols")));
+ }
+ else
+ {
+ /*
+ * currently in the binary protocol we can only emit encoded
+ * datums in the server encoding. There's no support for encoding
+ * conversion.
+ */
+ if (wanted_encoding != GetDatabaseEncoding())
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("encoding conversion for binary datum not supported yet"),
+ errdetail("expected_encoding %s must be unset or match server_encoding %s",
+ data->client_expected_encoding, GetDatabaseEncodingName())));
+ }
+
+ data->field_datum_encoding = wanted_encoding;
+ }
+
+ /*
+ * It's obviously not possible to send binary representatio of data
+ * unless we use the binary output.
+ */
+ if (opt->output_type == OUTPUT_PLUGIN_BINARY_OUTPUT &&
+ data->client_want_internal_basetypes)
+ {
+ data->allow_internal_basetypes =
+ check_binary_compatibility(data);
+ }
+
+ if (opt->output_type == OUTPUT_PLUGIN_BINARY_OUTPUT &&
+ data->client_want_binary_basetypes &&
+ data->client_binary_basetypes_major_version == PG_VERSION_NUM / 100)
+ {
+ data->allow_binary_basetypes = true;
+ }
+
+ /*
+ * Will we forward changesets? We have to if we're on 9.4;
+ * otherwise honour the client's request.
+ */
+ if (PG_VERSION_NUM/100 == 904)
+ {
+ /*
+ * 9.4 unconditionally forwards changesets due to lack of
+ * replication origins, and it can't ever send origin info
+ * for the same reason.
+ */
+ data->forward_changesets = true;
+ data->forward_changeset_origins = false;
+
+ if (data->client_forward_changesets_set
+ && !data->client_forward_changesets)
+ {
+ ereport(DEBUG1,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("Cannot disable changeset forwarding on PostgreSQL 9.4")));
+ }
+ }
+ else if (data->client_forward_changesets_set
+ && data->client_forward_changesets)
+ {
+ /* Client explicitly asked for forwarding; forward csets and origins */
+ data->forward_changesets = true;
+ data->forward_changeset_origins = true;
+ }
+ else
+ {
+ /* Default to not forwarding or honour client's request not to fwd */
+ data->forward_changesets = false;
+ data->forward_changeset_origins = false;
+ }
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+
+ data->hooks_mctxt = AllocSetContextCreate(ctx->context,
+ "pglogical_output hooks context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+ load_hooks(data);
+ call_startup_hook(data, ctx->output_plugin_options);
+ }
+ }
+}
+
+/*
+ * BEGIN callback
+ */
+void
+pg_decode_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+ bool send_replication_origin = data->forward_changeset_origins;
+
+ if (!startup_message_sent)
+ send_startup_message(ctx, data, false /* can't be last message */);
+
+ /* If the record didn't originate locally, send origin info */
+ send_replication_origin &= txn->origin_id != InvalidRepOriginId;
+
+ OutputPluginPrepareWrite(ctx, !send_replication_origin);
+ data->api->write_begin(ctx->out, data, txn);
+
+ if (send_replication_origin)
+ {
+ char *origin;
+
+ /* Message boundary */
+ OutputPluginWrite(ctx, false);
+ OutputPluginPrepareWrite(ctx, true);
+
+ /*
+ * XXX: which behaviour we want here?
+ *
+ * Alternatives:
+ * - don't send origin message if origin name not found
+ * (that's what we do now)
+ * - throw error - that will break replication, not good
+ * - send some special "unknown" origin
+ */
+ if (data->api->write_origin &&
+ replorigin_by_oid(txn->origin_id, true, &origin))
+ data->api->write_origin(ctx->out, origin, txn->origin_lsn);
+ }
+
+ OutputPluginWrite(ctx, true);
+}
+
+/*
+ * COMMIT callback
+ */
+void
+pg_decode_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_commit(ctx->out, data, txn, commit_lsn);
+ OutputPluginWrite(ctx, true);
+}
+
+void
+pg_decode_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ Relation relation, ReorderBufferChange *change)
+{
+ PGLogicalOutputData *data = ctx->output_plugin_private;
+ MemoryContext old;
+
+ /* First check the table filter */
+ if (!call_row_filter_hook(data, txn, relation, change))
+ return;
+
+ /* Avoid leaking memory by using and resetting our own context */
+ old = MemoryContextSwitchTo(data->context);
+
+ /* TODO: add caching (send only if changed) */
+ if (data->api->write_rel)
+ {
+ OutputPluginPrepareWrite(ctx, false);
+ data->api->write_rel(ctx->out, relation);
+ OutputPluginWrite(ctx, false);
+ }
+
+ /* Send the data */
+ switch (change->action)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_insert(ctx->out, data, relation,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ {
+ HeapTuple oldtuple = change->data.tp.oldtuple ?
+ &change->data.tp.oldtuple->tuple : NULL;
+
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_update(ctx->out, data, relation, oldtuple,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ }
+ case REORDER_BUFFER_CHANGE_DELETE:
+ if (change->data.tp.oldtuple)
+ {
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_delete(ctx->out, data, relation,
+ &change->data.tp.oldtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ }
+ else
+ elog(DEBUG1, "didn't send DELETE change because of missing oldtuple");
+ break;
+ default:
+ Assert(false);
+ }
+
+ /* Cleanup */
+ MemoryContextSwitchTo(old);
+ MemoryContextReset(data->context);
+}
+
+/*
+ * Decide if the whole transaction with specific origin should be filtered out.
+ */
+static bool
+pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id)
+{
+ PGLogicalOutputData *data = ctx->output_plugin_private;
+
+ if (!call_txn_filter_hook(data, origin_id))
+ return true;
+
+ if (!data->forward_changesets && origin_id != InvalidRepOriginId)
+ return true;
+
+ return false;
+}
+
+static void
+send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message)
+{
+ List *msg;
+
+ Assert(!startup_message_sent);
+
+ msg = prepare_startup_message(data);
+
+ /*
+ * We could free the extra_startup_params DefElem list here, but it's
+ * pretty harmless to just ignore it, since it's in the decoding memory
+ * context anyway, and we don't know if it's safe to free the defnames or
+ * not.
+ */
+
+ OutputPluginPrepareWrite(ctx, last_message);
+ data->api->write_startup_message(ctx->out, msg);
+ OutputPluginWrite(ctx, last_message);
+
+ pfree(msg);
+
+ startup_message_sent = true;
+}
+
+static void pg_decode_shutdown(LogicalDecodingContext * ctx)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+
+ call_shutdown_hook(data);
+
+ if (data->hooks_mctxt != NULL)
+ {
+ MemoryContextDelete(data->hooks_mctxt);
+ data->hooks_mctxt = NULL;
+ }
+}
diff --git a/contrib/pglogical_output/pglogical_output.h b/contrib/pglogical_output/pglogical_output.h
new file mode 100644
index 0000000..a874c40
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.h
@@ -0,0 +1,105 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.h
+ * pglogical output plugin
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_OUTPUT_H
+#define PG_LOGICAL_OUTPUT_H
+
+#include "nodes/parsenodes.h"
+
+#include "replication/logical.h"
+#include "replication/output_plugin.h"
+
+#include "storage/lock.h"
+
+#include "pglogical_output/hooks.h"
+
+#include "pglogical_proto.h"
+
+#define PG_LOGICAL_PROTO_VERSION_NUM 1
+#define PG_LOGICAL_PROTO_MIN_VERSION_NUM 1
+
+/*
+ * The name of a hook function. This is used instead of the usual List*
+ * because can serve as a hash key.
+ *
+ * Must be zeroed on allocation if used as a hash key since padding is
+ * *not* ignored on compare.
+ */
+typedef struct HookFuncName
+{
+ /* funcname is more likely to be unique, so goes first */
+ char function[NAMEDATALEN];
+ char schema[NAMEDATALEN];
+} HookFuncName;
+
+typedef struct PGLogicalOutputData
+{
+ MemoryContext context;
+
+ PGLogicalProtoAPI *api;
+
+ /* protocol */
+ bool allow_internal_basetypes;
+ bool allow_binary_basetypes;
+ bool forward_changesets;
+ bool forward_changeset_origins;
+ int field_datum_encoding;
+
+ /*
+ * client info
+ *
+ * Lots of this should move to a separate shorter-lived struct used only
+ * during parameter reading, since it contains what the client asked for.
+ * Once we've processed this during startup we don't refer to it again.
+ */
+ uint32 client_pg_version;
+ uint32 client_max_proto_version;
+ uint32 client_min_proto_version;
+ const char *client_expected_encoding;
+ const char *client_protocol_format;
+ uint32 client_binary_basetypes_major_version;
+ bool client_want_internal_basetypes_set;
+ bool client_want_internal_basetypes;
+ bool client_want_binary_basetypes_set;
+ bool client_want_binary_basetypes;
+ bool client_binary_bigendian_set;
+ bool client_binary_bigendian;
+ uint32 client_binary_sizeofdatum;
+ uint32 client_binary_sizeofint;
+ uint32 client_binary_sizeoflong;
+ bool client_binary_float4byval_set;
+ bool client_binary_float4byval;
+ bool client_binary_float8byval_set;
+ bool client_binary_float8byval;
+ bool client_binary_intdatetimes_set;
+ bool client_binary_intdatetimes;
+ bool client_forward_changesets_set;
+ bool client_forward_changesets;
+ bool client_no_txinfo;
+
+ /* hooks */
+ List *hooks_setup_funcname;
+ struct PGLogicalHooks hooks;
+ MemoryContext hooks_mctxt;
+
+ /* DefElem<String> list populated by startup hook */
+ List *extra_startup_params;
+} PGLogicalOutputData;
+
+typedef struct PGLogicalTupleData
+{
+ Datum values[MaxTupleAttributeNumber];
+ bool nulls[MaxTupleAttributeNumber];
+ bool changed[MaxTupleAttributeNumber];
+} PGLogicalTupleData;
+
+#endif /* PG_LOGICAL_OUTPUT_H */
diff --git a/contrib/pglogical_output/pglogical_output/README b/contrib/pglogical_output/pglogical_output/README
new file mode 100644
index 0000000..5480e5c
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/README
@@ -0,0 +1,7 @@
+/*
+ * This directory contains the public header files for the pglogical_output
+ * extension. It is installed into the PostgreSQL source tree when the extension
+ * is installed.
+ *
+ * These headers are not part of the PostgreSQL project its self.
+ */
diff --git a/contrib/pglogical_output/pglogical_output/hooks.h b/contrib/pglogical_output/pglogical_output/hooks.h
new file mode 100644
index 0000000..b20fa72
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/hooks.h
@@ -0,0 +1,72 @@
+#ifndef PGLOGICAL_OUTPUT_HOOKS_H
+#define PGLOGICAL_OUTPUT_HOOKS_H
+
+#include "access/xlogdefs.h"
+#include "nodes/pg_list.h"
+#include "utils/rel.h"
+#include "utils/palloc.h"
+#include "replication/reorderbuffer.h"
+
+struct PGLogicalOutputData;
+typedef struct PGLogicalOutputData PGLogicalOutputData;
+
+/*
+ * This header is to be included by extensions that implement pglogical output
+ * plugin callback hooks for transaction origin and row filtering, etc. It is
+ * installed as "pglogical_output/hooks.h"
+ *
+ * See the README.md and the example in examples/hooks/ for details on hooks.
+ */
+
+
+struct PGLogicalStartupHookArgs
+{
+ void *private_data;
+ List *in_params;
+ List *out_params;
+};
+
+typedef void (*pglogical_startup_hook_fn)(struct PGLogicalStartupHookArgs *args);
+
+
+struct PGLogicalTxnFilterArgs
+{
+ void *private_data;
+ RepOriginId origin_id;
+};
+
+typedef bool (*pglogical_txn_filter_hook_fn)(struct PGLogicalTxnFilterArgs *args);
+
+
+struct PGLogicalRowFilterArgs
+{
+ void *private_data;
+ Relation changed_rel;
+ enum ReorderBufferChangeType change_type;
+};
+
+typedef bool (*pglogical_row_filter_hook_fn)(struct PGLogicalRowFilterArgs *args);
+
+
+struct PGLogicalShutdownHookArgs
+{
+ void *private_data;
+};
+
+typedef void (*pglogical_shutdown_hook_fn)(struct PGLogicalShutdownHookArgs *args);
+
+/*
+ * This struct is passed to the pglogical_get_hooks_fn as the first argument,
+ * typed 'internal', and is unwrapped with `DatumGetPointer`.
+ */
+struct PGLogicalHooks
+{
+ pglogical_startup_hook_fn startup_hook;
+ pglogical_shutdown_hook_fn shutdown_hook;
+ pglogical_txn_filter_hook_fn txn_filter_hook;
+ pglogical_row_filter_hook_fn row_filter_hook;
+ void *hooks_private_data;
+};
+
+
+#endif /* PGLOGICAL_OUTPUT_HOOKS_H */
diff --git a/contrib/pglogical_output/pglogical_proto.c b/contrib/pglogical_output/pglogical_proto.c
new file mode 100644
index 0000000..47a883f
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.c
@@ -0,0 +1,49 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.c
+ * pglogical protocol functions
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+#include "pglogical_proto_native.h"
+#include "pglogical_proto_json.h"
+
+PGLogicalProtoAPI *
+pglogical_init_api(PGLogicalProtoType typ)
+{
+ PGLogicalProtoAPI *res = palloc0(sizeof(PGLogicalProtoAPI));
+
+ if (typ == PGLogicalProtoJson)
+ {
+ res->write_rel = NULL;
+ res->write_begin = pglogical_json_write_begin;
+ res->write_commit = pglogical_json_write_commit;
+ res->write_origin = NULL;
+ res->write_insert = pglogical_json_write_insert;
+ res->write_update = pglogical_json_write_update;
+ res->write_delete = pglogical_json_write_delete;
+ res->write_startup_message = json_write_startup_message;
+ }
+ else
+ {
+ res->write_rel = pglogical_write_rel;
+ res->write_begin = pglogical_write_begin;
+ res->write_commit = pglogical_write_commit;
+ res->write_origin = pglogical_write_origin;
+ res->write_insert = pglogical_write_insert;
+ res->write_update = pglogical_write_update;
+ res->write_delete = pglogical_write_delete;
+ res->write_startup_message = write_startup_message;
+ }
+
+ return res;
+}
diff --git a/contrib/pglogical_output/pglogical_proto.h b/contrib/pglogical_output/pglogical_proto.h
new file mode 100644
index 0000000..b56ff6f
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.h
@@ -0,0 +1,57 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.h
+ * pglogical protocol
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_H
+#define PG_LOGICAL_PROTO_H
+
+typedef void (*pglogical_write_rel_fn)(StringInfo out, Relation rel);
+
+typedef void (*pglogical_write_begin_fn)(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+typedef void (*pglogical_write_commit_fn)(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+typedef void (*pglogical_write_origin_fn)(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn);
+
+typedef void (*pglogical_write_insert_fn)(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+typedef void (*pglogical_write_update_fn)(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+typedef void (*pglogical_write_delete_fn)(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+typedef void (*write_startup_message_fn)(StringInfo out, List *msg);
+
+typedef struct PGLogicalProtoAPI
+{
+ pglogical_write_rel_fn write_rel;
+ pglogical_write_begin_fn write_begin;
+ pglogical_write_commit_fn write_commit;
+ pglogical_write_origin_fn write_origin;
+ pglogical_write_insert_fn write_insert;
+ pglogical_write_update_fn write_update;
+ pglogical_write_delete_fn write_delete;
+ write_startup_message_fn write_startup_message;
+} PGLogicalProtoAPI;
+
+
+typedef enum PGLogicalProtoType
+{
+ PGLogicalProtoNative,
+ PGLogicalProtoJson
+} PGLogicalProtoType;
+
+extern PGLogicalProtoAPI *pglogical_init_api(PGLogicalProtoType typ);
+
+#endif /* PG_LOGICAL_PROTO_H */
diff --git a/contrib/pglogical_output/pglogical_proto_json.c b/contrib/pglogical_output/pglogical_proto_json.c
new file mode 100644
index 0000000..a7a7686
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_json.c
@@ -0,0 +1,198 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_json.c
+ * pglogical protocol functions for json support
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_json.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto_json.h"
+
+#include "access/sysattr.h"
+#include "access/tuptoaster.h"
+#include "access/xact.h"
+
+#include "catalog/catversion.h"
+#include "catalog/index.h"
+
+#include "catalog/namespace.h"
+#include "catalog/pg_class.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_namespace.h"
+#include "catalog/pg_type.h"
+
+#include "commands/dbcommands.h"
+
+#include "executor/spi.h"
+
+#include "libpq/pqformat.h"
+
+#include "mb/pg_wchar.h"
+
+#include "replication/origin.h"
+
+#include "utils/builtins.h"
+#include "utils/json.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/typcache.h"
+
+
+/*
+ * Write BEGIN to the output stream.
+ */
+void
+pglogical_json_write_begin(StringInfo out, PGLogicalOutputData *data, ReorderBufferTXN *txn)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfoString(out, "\"action\":\"B\"");
+ appendStringInfo(out, ", has_catalog_changes:\"%c\"",
+ txn->has_catalog_changes ? 't' : 'f');
+ if (txn->origin_id != InvalidRepOriginId)
+ appendStringInfo(out, ", origin_id:\"%u\"", txn->origin_id);
+ if (!data->client_no_txinfo)
+ {
+ appendStringInfo(out, ", xid:\"%u\"", txn->xid);
+ appendStringInfo(out, ", first_lsn:\"%X/%X\"",
+ (uint32)(txn->first_lsn >> 32), (uint32)(txn->first_lsn));
+ appendStringInfo(out, ", origin_lsn:\"%X/%X\"",
+ (uint32)(txn->origin_lsn >> 32), (uint32)(txn->origin_lsn));
+ if (txn->commit_time != 0)
+ appendStringInfo(out, ", commit_time:\"%s\"",
+ timestamptz_to_str(txn->commit_time));
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write COMMIT to the output stream.
+ */
+void
+pglogical_json_write_commit(StringInfo out, PGLogicalOutputData *data, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfoString(out, "{\"action\":\"C\"}");
+ if (!data->client_no_txinfo)
+ {
+ appendStringInfo(out, ", final_lsn:\"%X/%X\"",
+ (uint32)(txn->final_lsn >> 32), (uint32)(txn->final_lsn));
+ appendStringInfo(out, ", end_lsn:\"%X/%X\"",
+ (uint32)(txn->end_lsn >> 32), (uint32)(txn->end_lsn));
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write a tuple to the outputstream, in the most efficient format possible.
+ */
+static void
+json_write_tuple(StringInfo out, Relation rel, HeapTuple tuple)
+{
+ TupleDesc desc;
+ Datum tupdatum,
+ json;
+
+ desc = RelationGetDescr(rel);
+ tupdatum = heap_copy_tuple_as_datum(tuple, desc);
+ json = DirectFunctionCall1(row_to_json, tupdatum);
+
+ appendStringInfoString(out, TextDatumGetCString(json));
+}
+
+/*
+ * Write change.
+ *
+ * Generic function handling DML changes.
+ */
+static void
+pglogical_json_write_change(StringInfo out, const char *change, Relation rel,
+ HeapTuple oldtuple, HeapTuple newtuple)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfo(out, "\"action\":\"%s\",\"relation\":[\"%s\",\"%s\"]",
+ change,
+ get_namespace_name(RelationGetNamespace(rel)),
+ RelationGetRelationName(rel));
+
+ if (oldtuple)
+ {
+ appendStringInfoString(out, ",\"oldtuple\":");
+ json_write_tuple(out, rel, oldtuple);
+ }
+ if (newtuple)
+ {
+ appendStringInfoString(out, ",\"newtuple\":");
+ json_write_tuple(out, rel, newtuple);
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write INSERT to the output stream.
+ */
+void
+pglogical_json_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple)
+{
+ pglogical_json_write_change(out, "I", rel, NULL, newtuple);
+}
+
+/*
+ * Write UPDATE to the output stream.
+ */
+void
+pglogical_json_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple)
+{
+ pglogical_json_write_change(out, "U", rel, oldtuple, newtuple);
+}
+
+/*
+ * Write DELETE to the output stream.
+ */
+void
+pglogical_json_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple)
+{
+ pglogical_json_write_change(out, "D", rel, oldtuple, NULL);
+}
+
+/*
+ * The startup message should be constructed as a json object, one
+ * key/value per DefElem list member.
+ */
+void
+json_write_startup_message(StringInfo out, List *msg)
+{
+ ListCell *lc;
+ bool first = true;
+
+ appendStringInfoString(out, "{\"action\":\"S\", \"params\": {");
+ foreach (lc, msg)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ if (first)
+ first = false;
+ else
+ appendStringInfoChar(out, ',');
+ escape_json(out, param->defname);
+ appendStringInfoChar(out, ':');
+ escape_json(out, strVal(param->arg));
+ }
+ appendStringInfoString(out, "}}");
+}
diff --git a/contrib/pglogical_output/pglogical_proto_json.h b/contrib/pglogical_output/pglogical_proto_json.h
new file mode 100644
index 0000000..d853e9e
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_json.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_json.h
+ * pglogical protocol, json implementation
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_json.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_JSON_H
+#define PG_LOGICAL_PROTO_JSON_H
+
+
+extern void pglogical_json_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+extern void pglogical_json_write_commit(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+extern void pglogical_json_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+extern void pglogical_json_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+extern void pglogical_json_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+extern void json_write_startup_message(StringInfo out, List *msg);
+
+#endif /* PG_LOGICAL_PROTO_JSON_H */
diff --git a/contrib/pglogical_output/pglogical_proto_native.c b/contrib/pglogical_output/pglogical_proto_native.c
new file mode 100644
index 0000000..baaf324
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_native.c
@@ -0,0 +1,494 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_native.c
+ * pglogical binary protocol functions
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_native.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto_native.h"
+
+#include "access/sysattr.h"
+#include "access/tuptoaster.h"
+#include "access/xact.h"
+
+#include "catalog/catversion.h"
+#include "catalog/index.h"
+
+#include "catalog/namespace.h"
+#include "catalog/pg_class.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_namespace.h"
+#include "catalog/pg_type.h"
+
+#include "commands/dbcommands.h"
+
+#include "executor/spi.h"
+
+#include "libpq/pqformat.h"
+
+#include "mb/pg_wchar.h"
+
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/typcache.h"
+
+#define IS_REPLICA_IDENTITY 1
+
+static void pglogical_write_attrs(StringInfo out, Relation rel);
+static void pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple);
+static char decide_datum_transfer(Form_pg_attribute att,
+ Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes);
+
+/*
+ * Write relation description to the output stream.
+ */
+void
+pglogical_write_rel(StringInfo out, Relation rel)
+{
+ const char *nspname;
+ uint8 nspnamelen;
+ const char *relname;
+ uint8 relnamelen;
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'R'); /* sending RELATION */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ nspname = get_namespace_name(rel->rd_rel->relnamespace);
+ if (nspname == NULL)
+ elog(ERROR, "cache lookup failed for namespace %u",
+ rel->rd_rel->relnamespace);
+ nspnamelen = strlen(nspname) + 1;
+
+ relname = NameStr(rel->rd_rel->relname);
+ relnamelen = strlen(relname) + 1;
+
+ pq_sendbyte(out, nspnamelen); /* schema name length */
+ pq_sendbytes(out, nspname, nspnamelen);
+
+ pq_sendbyte(out, relnamelen); /* table name length */
+ pq_sendbytes(out, relname, relnamelen);
+
+ /* send the attribute info */
+ pglogical_write_attrs(out, rel);
+}
+
+/*
+ * Write relation attributes to the outputstream.
+ */
+static void
+pglogical_write_attrs(StringInfo out, Relation rel)
+{
+ TupleDesc desc;
+ int i;
+ uint16 nliveatts = 0;
+ Bitmapset *idattrs;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'A'); /* sending ATTRS */
+
+ /* send number of live attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* fetch bitmap of REPLICATION IDENTITY attributes */
+ idattrs = RelationGetIndexAttrBitmap(rel, INDEX_ATTR_BITMAP_IDENTITY_KEY);
+
+ /* send the attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ Form_pg_attribute att = desc->attrs[i];
+ uint8 flags = 0;
+ uint16 len;
+ const char *attname;
+
+ if (att->attisdropped)
+ continue;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ idattrs))
+ flags |= IS_REPLICA_IDENTITY;
+
+ pq_sendbyte(out, 'C'); /* column definition follows */
+ pq_sendbyte(out, flags);
+
+ pq_sendbyte(out, 'N'); /* column name block follows */
+ attname = NameStr(att->attname);
+ len = strlen(attname) + 1;
+ pq_sendint(out, len, 2);
+ pq_sendbytes(out, attname, len); /* data */
+ }
+}
+
+/*
+ * Write BEGIN to the output stream.
+ */
+void
+pglogical_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'B'); /* BEGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, txn->final_lsn);
+ pq_sendint64(out, txn->commit_time);
+ pq_sendint(out, txn->xid, 4);
+}
+
+/*
+ * Write COMMIT to the output stream.
+ */
+void
+pglogical_write_commit(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'C'); /* sending COMMIT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* send fixed fields */
+ pq_sendint64(out, commit_lsn);
+ pq_sendint64(out, txn->end_lsn);
+ pq_sendint64(out, txn->commit_time);
+}
+
+/*
+ * Write ORIGIN to the output stream.
+ */
+void
+pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn)
+{
+ uint8 flags = 0;
+ uint8 len;
+
+ Assert(strlen(origin) < 255);
+
+ pq_sendbyte(out, 'O'); /* ORIGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, origin_lsn);
+
+ /* origin */
+ len = strlen(origin) + 1;
+ pq_sendbyte(out, len);
+ pq_sendbytes(out, origin, len);
+}
+
+/*
+ * Write INSERT to the output stream.
+ */
+void
+pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'I'); /* action INSERT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write UPDATE to the output stream.
+ */
+void
+pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'U'); /* action UPDATE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ if (oldtuple != NULL)
+ {
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+ }
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write DELETE to the output stream.
+ */
+void
+pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'D'); /* action DELETE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+}
+
+/*
+ * Most of the brains for startup message creation lives in
+ * pglogical_config.c, so this presently just sends the set of key/value pairs.
+ */
+void
+write_startup_message(StringInfo out, List *msg)
+{
+ ListCell *lc;
+
+ pq_sendbyte(out, 'S'); /* message type field */
+ pq_sendbyte(out, 1); /* startup message version */
+ foreach (lc, msg)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ /* null-terminated key and value pairs, in client_encoding */
+ pq_sendstring(out, param->defname);
+ pq_sendstring(out, strVal(param->arg));
+ }
+}
+
+/*
+ * Write a tuple to the outputstream, in the most efficient format possible.
+ */
+static void
+pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple)
+{
+ TupleDesc desc;
+ Datum values[MaxTupleAttributeNumber];
+ bool isnull[MaxTupleAttributeNumber];
+ int i;
+ uint16 nliveatts = 0;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'T'); /* sending TUPLE */
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* try to allocate enough memory from the get go */
+ enlargeStringInfo(out, tuple->t_len +
+ nliveatts * (1 + 4));
+
+ /*
+ * XXX: should this prove to be a relevant bottleneck, it might be
+ * interesting to inline heap_deform_tuple() here, we don't actually need
+ * the information in the form we get from it.
+ */
+ heap_deform_tuple(tuple, desc, values, isnull);
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ HeapTuple typtup;
+ Form_pg_type typclass;
+ Form_pg_attribute att = desc->attrs[i];
+ char transfer_type;
+
+ /* skip dropped columns */
+ if (att->attisdropped)
+ continue;
+
+ if (isnull[i])
+ {
+ pq_sendbyte(out, 'n'); /* null column */
+ continue;
+ }
+ else if (att->attlen == -1 && VARATT_IS_EXTERNAL_ONDISK(values[i]))
+ {
+ pq_sendbyte(out, 'u'); /* unchanged toast column */
+ continue;
+ }
+
+ typtup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid));
+ if (!HeapTupleIsValid(typtup))
+ elog(ERROR, "cache lookup failed for type %u", att->atttypid);
+ typclass = (Form_pg_type) GETSTRUCT(typtup);
+
+ transfer_type = decide_datum_transfer(att, typclass,
+ data->allow_internal_basetypes,
+ data->allow_binary_basetypes);
+
+ switch (transfer_type)
+ {
+ case 'i':
+ pq_sendbyte(out, 'i'); /* internal-format binary data follows */
+
+ /* pass by value */
+ if (att->attbyval)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ enlargeStringInfo(out, att->attlen);
+ store_att_byval(out->data + out->len, values[i],
+ att->attlen);
+ out->len += att->attlen;
+ out->data[out->len] = '\0';
+ }
+ /* fixed length non-varlena pass-by-reference type */
+ else if (att->attlen > 0)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ appendBinaryStringInfo(out, DatumGetPointer(values[i]),
+ att->attlen);
+ }
+ /* varlena type */
+ else if (att->attlen == -1)
+ {
+ char *data = DatumGetPointer(values[i]);
+
+ /* send indirect datums inline */
+ if (VARATT_IS_EXTERNAL_INDIRECT(values[i]))
+ {
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, data);
+ data = (char *) redirect.pointer;
+ }
+
+ Assert(!VARATT_IS_EXTERNAL(data));
+
+ pq_sendint(out, VARSIZE_ANY(data), 4); /* length */
+
+ appendBinaryStringInfo(out, data, VARSIZE_ANY(data));
+ }
+ else
+ elog(ERROR, "unsupported tuple type");
+
+ break;
+
+ case 'b':
+ {
+ bytea *outputbytes;
+ int len;
+
+ pq_sendbyte(out, 'b'); /* binary send/recv data follows */
+
+ outputbytes = OidSendFunctionCall(typclass->typsend,
+ values[i]);
+
+ len = VARSIZE(outputbytes) - VARHDRSZ;
+ pq_sendint(out, len, 4); /* length */
+ pq_sendbytes(out, VARDATA(outputbytes), len); /* data */
+ pfree(outputbytes);
+ }
+ break;
+
+ default:
+ {
+ char *outputstr;
+ int len;
+
+ pq_sendbyte(out, 't'); /* 'text' data follows */
+
+ outputstr = OidOutputFunctionCall(typclass->typoutput,
+ values[i]);
+ len = strlen(outputstr) + 1;
+ pq_sendint(out, len, 4); /* length */
+ appendBinaryStringInfo(out, outputstr, len); /* data */
+ pfree(outputstr);
+ }
+ }
+
+ ReleaseSysCache(typtup);
+ }
+}
+
+/*
+ * Make the executive decision about which protocol to use.
+ */
+static char
+decide_datum_transfer(Form_pg_attribute att, Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes)
+{
+ /*
+ * Use the binary protocol, if allowed, for builtin & plain datatypes.
+ */
+ if (allow_internal_basetypes &&
+ typclass->typtype == 'b' &&
+ att->atttypid < FirstNormalObjectId &&
+ typclass->typelem == InvalidOid)
+ {
+ return 'i';
+ }
+ /*
+ * Use send/recv, if allowed, if the type is plain or builtin.
+ *
+ * XXX: we can't use send/recv for array or composite types for now due to
+ * the embedded oids.
+ */
+ else if (allow_binary_basetypes &&
+ OidIsValid(typclass->typreceive) &&
+ (att->atttypid < FirstNormalObjectId || typclass->typtype != 'c') &&
+ (att->atttypid < FirstNormalObjectId || typclass->typelem == InvalidOid))
+ {
+ return 'b';
+ }
+
+ return 't';
+}
diff --git a/contrib/pglogical_output/pglogical_proto_native.h b/contrib/pglogical_output/pglogical_proto_native.h
new file mode 100644
index 0000000..729bee0
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_native.h
@@ -0,0 +1,37 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_native.h
+ * pglogical protocol, native implementation
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_native.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_NATIVE_H
+#define PG_LOGICAL_PROTO_NATIVE_H
+
+
+extern void pglogical_write_rel(StringInfo out, Relation rel);
+
+extern void pglogical_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+extern void pglogical_write_commit(StringInfo out,PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+extern void pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn);
+
+extern void pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+extern void pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+extern void pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+extern void write_startup_message(StringInfo out, List *msg);
+
+#endif /* PG_LOGICAL_PROTO_NATIVE_H */
diff --git a/contrib/pglogical_output/regression.conf b/contrib/pglogical_output/regression.conf
new file mode 100644
index 0000000..367f706
--- /dev/null
+++ b/contrib/pglogical_output/regression.conf
@@ -0,0 +1,2 @@
+wal_level = logical
+max_replication_slots = 4
diff --git a/contrib/pglogical_output/sql/basic_json.sql b/contrib/pglogical_output/sql/basic_json.sql
new file mode 100644
index 0000000..11db1f5
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_json.sql
@@ -0,0 +1,14 @@
+\i sql/basic_setup.sql
+
+-- Simple decode with text-format tuples
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+\i sql/basic_teardown.sql
diff --git a/contrib/pglogical_output/sql/basic_native.sql b/contrib/pglogical_output/sql/basic_native.sql
new file mode 100644
index 0000000..5b6ec4b
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_native.sql
@@ -0,0 +1,27 @@
+\i sql/basic_setup.sql
+
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+
+\i sql/basic_teardown.sql
diff --git a/contrib/pglogical_output/sql/basic_setup.sql b/contrib/pglogical_output/sql/basic_setup.sql
new file mode 100644
index 0000000..19e154c
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_setup.sql
@@ -0,0 +1,62 @@
+SET synchronous_commit = on;
+
+-- Schema setup
+
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Queue up some work to decode with a variety of types
+
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+
+
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
diff --git a/contrib/pglogical_output/sql/basic_teardown.sql b/contrib/pglogical_output/sql/basic_teardown.sql
new file mode 100644
index 0000000..d7a752f
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_teardown.sql
@@ -0,0 +1,4 @@
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/sql/encoding_json.sql b/contrib/pglogical_output/sql/encoding_json.sql
new file mode 100644
index 0000000..543c306
--- /dev/null
+++ b/contrib/pglogical_output/sql/encoding_json.sql
@@ -0,0 +1,58 @@
+SET synchronous_commit = on;
+
+-- This file doesn't share common setup with the native tests,
+-- since it's specific to how the text protocol handles encodings.
+
+CREATE TABLE enctest(blah text);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+
+SET client_encoding = 'UTF-8';
+INSERT INTO enctest(blah)
+VALUES
+('áàä'),('fl'), ('½⅓'), ('カンジ');
+RESET client_encoding;
+
+
+SET client_encoding = 'LATIN-1';
+
+-- Will ERROR, explicit encoding request doesn't match client_encoding
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+-- Will succeed since we don't request any encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+-- Will succeed since it matches the current encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'LATIN-1',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+RESET client_encoding;
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE enctest;
diff --git a/contrib/pglogical_output/sql/hooks_json.sql b/contrib/pglogical_output/sql/hooks_json.sql
new file mode 100644
index 0000000..91a123e
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_json.sql
@@ -0,0 +1,28 @@
+\i sql/hooks_setup.sql
+
+-- Test table filter
+SELECT data FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+-- test action filter
+SELECT data FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+\i sql/hooks_teardown.sql
diff --git a/contrib/pglogical_output/sql/hooks_native.sql b/contrib/pglogical_output/sql/hooks_native.sql
new file mode 100644
index 0000000..e2bfc54
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_native.sql
@@ -0,0 +1,48 @@
+\i sql/hooks_setup.sql
+
+-- Regular hook setup
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+
+-- Test action filter
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+
+-- Invalid row fiter hook function
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+
+-- Hook filter functoin with wrong signature
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+
+\i sql/hooks_teardown.sql
diff --git a/contrib/pglogical_output/sql/hooks_setup.sql b/contrib/pglogical_output/sql/hooks_setup.sql
new file mode 100644
index 0000000..4de15b7
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_setup.sql
@@ -0,0 +1,37 @@
+CREATE EXTENSION pglogical_output_plhooks;
+
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
diff --git a/contrib/pglogical_output/sql/hooks_teardown.sql b/contrib/pglogical_output/sql/hooks_teardown.sql
new file mode 100644
index 0000000..837e2d0
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_teardown.sql
@@ -0,0 +1,10 @@
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/sql/params_native.sql b/contrib/pglogical_output/sql/params_native.sql
new file mode 100644
index 0000000..8b08732
--- /dev/null
+++ b/contrib/pglogical_output/sql/params_native.sql
@@ -0,0 +1,95 @@
+SET synchronous_commit = on;
+
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+--
+-- Various invalid parameter combos:
+--
+
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
diff --git a/contrib/pglogical_output/sql/pre_clean.sql b/contrib/pglogical_output/sql/pre_clean.sql
new file mode 100644
index 0000000..5c12dc0
--- /dev/null
+++ b/contrib/pglogical_output/sql/pre_clean.sql
@@ -0,0 +1,10 @@
+DO
+LANGUAGE plpgsql
+$$
+BEGIN
+ IF EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'regression_slot')
+ THEN
+ PERFORM pg_drop_replication_slot('regression_slot');
+ END IF;
+END;
+$$;
diff --git a/contrib/pglogical_output_plhooks/.gitignore b/contrib/pglogical_output_plhooks/.gitignore
new file mode 100644
index 0000000..140f8cf
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/.gitignore
@@ -0,0 +1 @@
+*.so
diff --git a/contrib/pglogical_output_plhooks/Makefile b/contrib/pglogical_output_plhooks/Makefile
new file mode 100644
index 0000000..ecd3f89
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/Makefile
@@ -0,0 +1,13 @@
+MODULES = pglogical_output_plhooks
+EXTENSION = pglogical_output_plhooks
+DATA = pglogical_output_plhooks--1.0.sql
+DOCS = README.pglogical_output_plhooks
+
+subdir = contrib/pglogical_output_plhooks
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# Allow the hook plugin to see the pglogical_output headers
+# Necessary because !PGXS builds don't respect PG_CPPFLAGS
+override CPPFLAGS := $(CPPFLAGS) -I$(top_srcdir)/contrib/pglogical_output
diff --git a/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks b/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
new file mode 100644
index 0000000..4f0ce81
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
@@ -0,0 +1,160 @@
+pglogical_output_plhooks is an example module for pglogical_output, showing how
+hooks can be implemented.
+
+It provides C wrappers to allow hooks to be written in any supported PL,
+such as PL/PgSQL.
+
+No effort is made to be efficient. To avoid the need to set up cache
+invalidation handling function calls are done via oid each time, with no
+FmgrInfo caching. Also, memory contexts are reset rather freely. If you
+want efficiency, write your hook in C.
+
+(Catalog timetravel is another reason not to write hooks in PLs; see below).
+
+Simple pointless example
+===
+
+To compile and install, just "make USE_PGXS=1 install". Note that pglogical
+must already be installed so that its headers can be found. You might have
+to set the `PATH` so that `pg_config` can be found.
+
+To use it:
+
+ CREATE EXTENSION pglogical_output_plhooks IN SCHEMA public;
+
+in the target database.
+
+Then create at least one hook procedure, of the supported hooks listed below.
+For the sake of this example we'll use some of the toy examples provided in the
+extension:
+
+* startup function: pglo_plhooks_demo_startup
+* row filter: pglo_plhooks_demo_row_filter
+* txn filter: pglo_plhooks_demo_txn_filter
+* shutdown function: pglo_plhooks_demo_shutdown
+
+Now add some arguments to your pglogical_output client's logical decoding setup
+parameters to specify the hook setup function and to tell
+pglogical_output_plhooks about one or more of the hooks you wish it to run. For
+example you might add the following parameters:
+
+ hooks.setup_function, public.pglo_plhooks_setup_fn,
+ pglo_plhooks.startup_hook, pglo_plhooks_demo_startup,
+ pglo_plhooks.row_filter_hook, pglo_plhooks_demo_row_filter,
+ pglo_plhooks.txn_filter_hook, pglo_plhooks_demo_txn_filter,
+ pglo_plhooks.shutdown_hook, pglo_plhooks_demo_shutdown,
+ pglo_plhooks.client_hook_arg, 'whatever-you-want'
+
+to configure the extension to load its hooks, then configure all the demo hooks.
+
+Why the preference for C hooks?
+===
+
+Speed. The row filter hook is called for *every single row* replicated.
+
+If a hook raises an ERROR then replication will probably stop. You won't be
+able to fix it either, because when you change the hook definition the new
+definition won't be visible in the catalogs at the current replay position due
+to catalog time travel. The old definition that raises an error will keep being
+used. You'll need to remove the problem hook from your logical decoding startup
+parameters, which will disable use the hook entirely, until replay proceeds
+past the point you fixed the problem with the hook function.
+
+Similarly, if you try to add use of a newly defined hook on an existing
+replication slot that hasn't replayed past the point you defined the hook yet,
+you'll get an error complaining that the hook function doesn't exist. Even
+though it clearly does when you look at it in psql. The reason is the same: in
+the time traveled catalogs it really doesn't exist. You have to replay past the
+point the hook was created then enable it. In this case the
+pglogical_output_plhooks startup hook will actually see your functions, but
+fail when it tries to call them during decoding since they'll appear to have
+vanished.
+
+If you write your hooks in C you can redefine them rather more easily, since
+the function definition is not subject to catalog timetravel. More importantly,
+it'll probably be a lot faster. The plhooks code has to do a lot of translation
+to pass information to the PL functions and more to get results back; it also
+has to do a lot of memory allocations and a memory context reset after each
+call. That all adds up.
+
+(You could actually write C functions to be called by this extension, but
+that'd be crazy.)
+
+Available hooks
+===
+
+The four hooks provided by pglogical_output are exposed by the module. See the
+pglogical_output documentation for details on what each hook does and when it
+runs.
+
+A function for each hook must have *exactly* the specified parameters and
+return value, or you'll get an error.
+
+None of the functions may return NULL. If they do you'll get an error.
+
+If you specified `pglo_plhooks.client_hook_arg` in the startup parameters it is
+passed as `client_hook_arg` to all hooks. If not specified the empty string is
+passed.
+
+You can find some toy examples in `pglogical_output_plhooks--1.0.sql`.
+
+
+
+Startup hook
+---
+
+Configured with `pglo_plhooks.startup_hook` startup parameter. Runs when
+logical decoding starts.
+
+Signature *must* be:
+
+ CREATE FUNCTION whatever_funcname(startup_params text[], client_hook_arg text)
+ RETURNS text[]
+
+startup_params is an array of the startup params passed to the pglogical output
+plugin, as alternating key/value elements in text representation.
+
+client_hook_arg is also passed.
+
+The return value is an array of alternating key/value elements forming a set
+of parameters you wish to add to the startup reply message sent by pglogical
+on decoding start. It must not be null; return `ARRAY[]::text[]` if you don't
+want to add any params.
+
+Transaction filter
+---
+
+Called only on 9.5+; ignored on 9.4.
+
+The arguments are the replication origin identifier and the client hook param.
+
+The return value is true to keep the transaction, false to discard it.
+
+Signature:
+
+ CREATE FUNCTION whatevername(origin_id int, client_hook_arg text)
+ RETURNS boolean
+
+Row filter
+--
+
+Called for each row. Return true to replicate the row, false to discard it.
+
+Arguments are the oid of the affected relation, and the change type: 'I'nsert,
+'U'pdate or 'D'elete. There is no way to access the change data - columns changed,
+new values, etc.
+
+Signature:
+
+ CREATE FUNCTION whatevername(affected_rel regclass, change_type "char", client_hook_arg text)
+ RETURNS boolean
+
+Shutdown hook
+--
+
+Pretty uninteresting, but included for completeness.
+
+Signature:
+
+ CREATE FUNCTION whatevername(client_hook_arg text)
+ RETURNS void
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql b/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
new file mode 100644
index 0000000..cdd2af3
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
@@ -0,0 +1,89 @@
+\echo Use "CREATE EXTENSION pglogical_output_plhooks" to load this file. \quit
+
+-- Use @extschema@ or leave search_path unchanged, don't use explicit schema
+
+CREATE FUNCTION pglo_plhooks_setup_fn(internal)
+RETURNS void
+STABLE
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+COMMENT ON FUNCTION pglo_plhooks_setup_fn(internal)
+IS 'Register pglogical output pl hooks. See docs for how to specify functions';
+
+--
+-- Called as the startup hook.
+--
+-- There's no useful way to expose the private data segment, so you
+-- just don't get to use that from pl hooks at this point. The C
+-- wrapper will extract a startup param named pglo_plhooks.client_hook_arg
+-- for you and pass it as client_hook_arg to all callbacks, though.
+--
+-- For implementation convenience, a null client_hook_arg is passed
+-- as the empty string.
+--
+-- Must return the empty array, not NULL, if it has nothing to add.
+--
+CREATE FUNCTION pglo_plhooks_demo_startup(startup_params text[], client_hook_arg text)
+RETURNS text[]
+LANGUAGE plpgsql AS $$
+DECLARE
+ elem text;
+ paramname text;
+ paramvalue text;
+BEGIN
+ FOREACH elem IN ARRAY startup_params
+ LOOP
+ IF elem IS NULL THEN
+ RAISE EXCEPTION 'Startup params may not be null';
+ END IF;
+
+ IF paramname IS NULL THEN
+ paramname := elem;
+ ELSIF paramvalue IS NULL THEN
+ paramvalue := elem;
+ ELSE
+ RAISE NOTICE 'got param: % = %', paramname, paramvalue;
+ paramname := NULL;
+ paramvalue := NULL;
+ END IF;
+ END LOOP;
+
+ RETURN ARRAY['pglo_plhooks_demo_startup_ran', 'true', 'otherparam', '42'];
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_txn_filter(origin_id int, client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- Not much to filter on really...
+ RAISE NOTICE 'Got tx with origin %',origin_id;
+ RETURN true;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_row_filter(affected_rel regclass, change_type "char", client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- This is a totally absurd test, since it checks if the upstream user
+ -- doing replication has rights to make modifications that have already
+ -- been committed and are being decoded for replication. Still, it shows
+ -- how the hook works...
+ IF pg_catalog.has_table_privilege(current_user, affected_rel,
+ CASE change_type WHEN 'I' THEN 'INSERT' WHEN 'U' THEN 'UPDATE' WHEN 'D' THEN 'DELETE' END)
+ THEN
+ RETURN true;
+ ELSE
+ RETURN false;
+ END IF;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_shutdown(client_hook_arg text)
+RETURNS void
+LANGUAGE plpgsql AS $$
+BEGIN
+ RAISE NOTICE 'Decoding shutdown';
+END;
+$$
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
new file mode 100644
index 0000000..a5144f0
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
@@ -0,0 +1,414 @@
+#include "postgres.h"
+
+#include "pglogical_output/hooks.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_type.h"
+
+#include "nodes/makefuncs.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/reorderbuffer.h"
+
+#include "utils/acl.h"
+#include "utils/array.h"
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+
+#include "fmgr.h"
+#include "miscadmin.h"
+
+PG_MODULE_MAGIC;
+
+PGDLLEXPORT extern Datum pglo_plhooks_setup_fn(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglo_plhooks_setup_fn);
+
+void pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args);
+void pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args);
+bool pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args);
+bool pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args);
+
+typedef struct PLHPrivate
+{
+ const char *client_arg;
+ Oid startup_hook;
+ Oid shutdown_hook;
+ Oid row_filter_hook;
+ Oid txn_filter_hook;
+ MemoryContext hook_call_context;
+} PLHPrivate;
+
+static void read_parameters(PLHPrivate *private, List *in_params);
+static Oid find_startup_hook(const char *proname);
+static Oid find_shutdown_hook(const char *proname);
+static Oid find_row_filter_hook(const char *proname);
+static Oid find_txn_filter_hook(const char *proname);
+static void exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params);
+
+void
+pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args)
+{
+ PLHPrivate *private;
+
+ /* pglogical_output promises to call us in a tx */
+ Assert(IsTransactionState());
+
+ /* Allocated in hook memory context, scoped to the logical decoding session: */
+ startup_args->private_data = private = (PLHPrivate*)palloc(sizeof(PLHPrivate));
+
+ private->startup_hook = InvalidOid;
+ private->shutdown_hook = InvalidOid;
+ private->row_filter_hook = InvalidOid;
+ private->txn_filter_hook = InvalidOid;
+ /* client_arg is the empty string when not specified to simplify function calls */
+ private->client_arg = "";
+
+ read_parameters(private, startup_args->in_params);
+
+ private->hook_call_context = AllocSetContextCreate(CurrentMemoryContext,
+ "pglogical_output plhooks hook call context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+
+ if (private->startup_hook != InvalidOid)
+ exec_user_startup_hook(private, startup_args->in_params, &startup_args->out_params);
+}
+
+void
+pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args)
+{
+ PLHPrivate *private = (PLHPrivate*)shutdown_args->private_data;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->shutdown_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo shutdown hook with %s", private->client_arg);
+ (void) OidFunctionCall1(
+ private->shutdown_hook,
+ CStringGetTextDatum(private->client_arg));
+ elog(DEBUG3, "called pglo shutdown hook");
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+}
+
+bool
+pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)rowfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->row_filter_hook))
+ {
+ char change_type;
+ switch (rowfilter_args->change_type)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ change_type = 'I';
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ change_type = 'U';
+ break;
+ case REORDER_BUFFER_CHANGE_DELETE:
+ change_type = 'D';
+ break;
+ default:
+ elog(ERROR, "unknown change type %d", rowfilter_args->change_type);
+ change_type = '0'; /* silence compiler */
+ }
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo row filter hook with (%u,%c,%s)",
+ rowfilter_args->changed_rel->rd_id, change_type,
+ private->client_arg);
+ ret = DatumGetBool(OidFunctionCall3(
+ private->row_filter_hook,
+ ObjectIdGetDatum(rowfilter_args->changed_rel->rd_id),
+ CharGetDatum(change_type),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "called pglo row filter hook, returns %d", (int)ret);
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+bool
+pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)txnfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+
+ if (OidIsValid(private->txn_filter_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ elog(DEBUG3, "calling pglo txn filter hook with (%hu,%s)",
+ txnfilter_args->origin_id, private->client_arg);
+ ret = DatumGetBool(OidFunctionCall2(
+ private->txn_filter_hook,
+ UInt16GetDatum(txnfilter_args->origin_id),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "calling pglo txn filter hook, returns %d", (int)ret);
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+Datum
+pglo_plhooks_setup_fn(PG_FUNCTION_ARGS)
+{
+ struct PGLogicalHooks *hooks = (struct PGLogicalHooks*) PG_GETARG_POINTER(0);
+
+ /* Your code doesn't need this, it's just for the tests: */
+ Assert(hooks != NULL);
+ Assert(hooks->hooks_private_data == NULL);
+ Assert(hooks->startup_hook == NULL);
+ Assert(hooks->shutdown_hook == NULL);
+ Assert(hooks->row_filter_hook == NULL);
+ Assert(hooks->txn_filter_hook == NULL);
+
+ /*
+ * Just assign the hook pointers. We're not meant to do much
+ * work here.
+ *
+ * Note that private_data is left untouched, to be set up by the
+ * startup hook.
+ */
+ hooks->startup_hook = pglo_plhooks_startup;
+ hooks->shutdown_hook = pglo_plhooks_shutdown;
+ hooks->row_filter_hook = pglo_plhooks_row_filter;
+ hooks->txn_filter_hook = pglo_plhooks_txn_filter;
+ elog(DEBUG3, "configured pglo hooks");
+
+ PG_RETURN_VOID();
+}
+
+static void
+exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params)
+{
+ ArrayType *startup_params;
+ Datum ret;
+ ListCell *lc;
+ Datum *startup_params_elems;
+ bool *startup_params_isnulls;
+ int n_startup_params;
+ int i;
+ MemoryContext old_ctx;
+
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ /*
+ * Build the input parameter array. NULL parameters are passed as the
+ * empty string for the sake of convenience. Each param is two
+ * elements, a key then a value element.
+ */
+ n_startup_params = list_length(in_params) * 2;
+ startup_params_elems = (Datum*)palloc0(sizeof(Datum)*n_startup_params);
+
+ i = 0;
+ foreach (lc, in_params)
+ {
+ DefElem * elem = (DefElem*)lfirst(lc);
+ const char *val;
+
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ val = "";
+ else
+ val = strVal(elem->arg);
+
+ startup_params_elems[i++] = CStringGetTextDatum(elem->defname);
+ startup_params_elems[i++] = CStringGetTextDatum(val);
+ }
+ Assert(i == n_startup_params);
+
+ startup_params = construct_array(startup_params_elems, n_startup_params,
+ TEXTOID, -1, false, 'i');
+
+ ret = OidFunctionCall2(
+ private->startup_hook,
+ PointerGetDatum(startup_params),
+ CStringGetTextDatum(private->client_arg));
+
+ /*
+ * deconstruct return array and add pairs of results to a DefElem list.
+ */
+ deconstruct_array(DatumGetArrayTypeP(ret), TEXTARRAYOID,
+ -1, false, 'i', &startup_params_elems, &startup_params_isnulls,
+ &n_startup_params);
+
+
+ *out_params = NIL;
+ for (i = 0; i < n_startup_params; i = i + 2)
+ {
+ char *value;
+ DefElem *elem;
+
+ if (startup_params_isnulls[i])
+ elog(ERROR, "Array entry corresponding to a key was null at idx=%d", i);
+
+ if (startup_params_isnulls[i+1])
+ value = "";
+ else
+ value = TextDatumGetCString(startup_params_elems[i+1]);
+
+ elem = makeDefElem(
+ TextDatumGetCString(startup_params_elems[i]),
+ (Node*)makeString(value));
+
+ *out_params = lcons(elem, *out_params);
+ }
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+}
+
+static void
+read_parameters(PLHPrivate *private, List *in_params)
+{
+ ListCell *option;
+
+ foreach(option, in_params)
+ {
+ DefElem *elem = lfirst(option);
+
+ if (pg_strcasecmp("pglo_plhooks.client_hook_arg", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.client_hook_arg may not be NULL");
+ private->client_arg = pstrdup(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.startup_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.startup_hook may not be NULL");
+ private->startup_hook = find_startup_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.shutdown_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.shutdown_hook may not be NULL");
+ private->shutdown_hook = find_shutdown_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.txn_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.txn_filter_hook may not be NULL");
+ private->txn_filter_hook = find_txn_filter_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.row_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.row_filter_hook may not be NULL");
+ private->row_filter_hook = find_row_filter_hook(strVal(elem->arg));
+ }
+ }
+}
+
+static Oid
+find_hook_fn(const char *funcname, Oid funcargtypes[], int nfuncargtypes, Oid returntype)
+{
+ Oid funcid;
+ List *qname;
+
+ qname = stringToQualifiedNameList(funcname);
+
+ /* find the the function */
+ funcid = LookupFuncName(qname, nfuncargtypes, funcargtypes, false);
+
+ /* Check expected return type */
+ if (get_func_rettype(funcid) != returntype)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s doesn't return expected type %d",
+ NameListToString(qname), returntype)));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(qname))));
+ }
+
+ list_free_deep(qname);
+
+ return funcid;
+}
+
+static Oid
+find_startup_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = TEXTARRAYOID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, VOIDOID);
+}
+
+static Oid
+find_shutdown_hook(const char *proname)
+{
+ Oid argtypes[1];
+
+ argtypes[0] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 1, VOIDOID);
+}
+
+static Oid
+find_row_filter_hook(const char *proname)
+{
+ Oid argtypes[3];
+
+ argtypes[0] = REGCLASSOID;
+ argtypes[1] = CHAROID;
+ argtypes[2] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 3, BOOLOID);
+}
+
+static Oid
+find_txn_filter_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = INT4OID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, BOOLOID);
+}
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
new file mode 100644
index 0000000..647b9ef
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
@@ -0,0 +1,4 @@
+comment = 'pglogical_output pl hooks'
+default_version = '1.0'
+module_pathname = '$libdir/pglogical_output_plhooks'
+relocatable = false
--
2.1.0
On 11/2/15 7:17 AM, Craig Ringer wrote:
The output plugin is suitable for a number of uses. It's designed
primarily to supply a data stream to drive a logical replication
client running in another PostgreSQL instance, like the pglogical
client discussed at PGConf.EU 2015.
So where is that client?
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 16 November 2015 at 09:57, Peter Eisentraut <peter_e@gmx.net> wrote:
On 11/2/15 7:17 AM, Craig Ringer wrote:
The output plugin is suitable for a number of uses. It's designed
primarily to supply a data stream to drive a logical replication
client running in another PostgreSQL instance, like the pglogical
client discussed at PGConf.EU 2015.So where is that client?
Not finished baking yet - in particular, the catalogs and UI are still in
flux. The time scale for getting that out is in the order of a few weeks.
The output plugin stands alone to a fair degree, especially with the json
output support. Comments would be greatly appreciated, especially from
people who're involved in replication, are currently using triggers to feed
data to external systems, etc.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
W dniu 12.11.2015, czw o godzinie 22∶23 +0800, użytkownik Craig Ringer
napisał:
Hi all
Here's an updated pglogical_output patch.
Selected changes since v1:
- add json protocol output support
- fix encoding comparisons to use parsed encoding not string name
- import protocol documentation
- significantly expand pg_regress tests
- move pglogical_output_plhooks to a top-level contrib
- remove 9.4 compatibility
I've just skimmed through the patch.
As you removed 9.4 compatibility - are mentions of 9.4 and 9.5
compatibility needed in README.md and README.plhooks?
It's not much text, but I'm not sure whether they shouldn't be
removed for 9.6-targeting change.
--
Tomasz Rybak GPG/PGP key ID: 2AD5 9860
Fingerprint A481 824E 7DD3 9C0E C40A 488E C654 FB33 2AD5 9860
http://member.acm.org/~tomaszrybak
On 19 November 2015 at 03:42, Tomasz Rybak <tomasz.rybak@post.pl> wrote:
I've just skimmed through the patch.
As you removed 9.4 compatibility - are mentions of 9.4 and 9.5
compatibility needed in README.md and README.plhooks?
Thanks. Removed.
Here's a v3 patch. Changes since v2:
- fix stray braces in commit json output
- significantly expand pg_regress tests
- quote all json keys
- validate all json in tests
- decode json startup message to a table
- filter out mutable keys in json startup message
- troubleshooting section of README covers using json
- documentation copy editing
Full patch against master attached. Diff since v2 at
While the downstream client using the binary protocol is still too WIP, the
json support is useful for all sorts of apps. It has all the same support
for hooks and filtering, origin information, etc. I'd love to hear from
anyone who's trying it out.
SGML conversion of docs still WIP.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
Attachments:
0001-Add-contrib-pglogical_output-a-logical-decoding-plug.patchtext/x-patch; charset=UTF-8; name=0001-Add-contrib-pglogical_output-a-logical-decoding-plug.patchDownload
From a3d21e70a9de7a1e5bb2093f592dde53bae707cb Mon Sep 17 00:00:00 2001
From: Craig Ringer <craig@2ndquadrant.com>
Date: Mon, 2 Nov 2015 19:34:21 +0800
Subject: [PATCH] Add contrib/pglogical_output, a logical decoding plugin
---
contrib/Makefile | 2 +
contrib/pglogical_output/.gitignore | 6 +
contrib/pglogical_output/Makefile | 27 +
contrib/pglogical_output/README.md | 622 +++++++++++++++++++++
contrib/pglogical_output/doc/.gitignore | 1 +
contrib/pglogical_output/doc/DESIGN.md | 124 ++++
contrib/pglogical_output/doc/protocol.txt | 546 ++++++++++++++++++
contrib/pglogical_output/expected/basic_json.out | 139 +++++
contrib/pglogical_output/expected/basic_json_1.out | 108 ++++
contrib/pglogical_output/expected/basic_native.out | 99 ++++
contrib/pglogical_output/expected/cleanup.out | 4 +
.../pglogical_output/expected/encoding_json.out | 59 ++
contrib/pglogical_output/expected/hooks_json.out | 202 +++++++
contrib/pglogical_output/expected/hooks_json_1.out | 139 +++++
contrib/pglogical_output/expected/hooks_native.out | 104 ++++
.../pglogical_output/expected/params_native.out | 118 ++++
.../pglogical_output/expected/params_native_1.out | 118 ++++
contrib/pglogical_output/expected/prep.out | 26 +
contrib/pglogical_output/pglogical_config.c | 499 +++++++++++++++++
contrib/pglogical_output/pglogical_config.h | 55 ++
contrib/pglogical_output/pglogical_hooks.c | 232 ++++++++
contrib/pglogical_output/pglogical_hooks.h | 22 +
contrib/pglogical_output/pglogical_output.c | 537 ++++++++++++++++++
contrib/pglogical_output/pglogical_output.h | 105 ++++
contrib/pglogical_output/pglogical_output/README | 7 +
contrib/pglogical_output/pglogical_output/hooks.h | 72 +++
contrib/pglogical_output/pglogical_proto.c | 49 ++
contrib/pglogical_output/pglogical_proto.h | 57 ++
contrib/pglogical_output/pglogical_proto_json.c | 204 +++++++
contrib/pglogical_output/pglogical_proto_json.h | 32 ++
contrib/pglogical_output/pglogical_proto_native.c | 494 ++++++++++++++++
contrib/pglogical_output/pglogical_proto_native.h | 37 ++
contrib/pglogical_output/regression.conf | 2 +
contrib/pglogical_output/sql/basic_json.sql | 24 +
contrib/pglogical_output/sql/basic_native.sql | 27 +
contrib/pglogical_output/sql/basic_setup.sql | 62 ++
contrib/pglogical_output/sql/basic_teardown.sql | 4 +
contrib/pglogical_output/sql/cleanup.sql | 4 +
contrib/pglogical_output/sql/encoding_json.sql | 58 ++
contrib/pglogical_output/sql/hooks_json.sql | 49 ++
contrib/pglogical_output/sql/hooks_native.sql | 48 ++
contrib/pglogical_output/sql/hooks_setup.sql | 37 ++
contrib/pglogical_output/sql/hooks_teardown.sql | 10 +
contrib/pglogical_output/sql/params_native.sql | 95 ++++
contrib/pglogical_output/sql/prep.sql | 30 +
contrib/pglogical_output_plhooks/.gitignore | 1 +
contrib/pglogical_output_plhooks/Makefile | 13 +
.../README.pglogical_output_plhooks | 158 ++++++
.../pglogical_output_plhooks--1.0.sql | 89 +++
.../pglogical_output_plhooks.c | 414 ++++++++++++++
.../pglogical_output_plhooks.control | 4 +
51 files changed, 5975 insertions(+)
create mode 100644 contrib/pglogical_output/.gitignore
create mode 100644 contrib/pglogical_output/Makefile
create mode 100644 contrib/pglogical_output/README.md
create mode 100644 contrib/pglogical_output/doc/.gitignore
create mode 100644 contrib/pglogical_output/doc/DESIGN.md
create mode 100644 contrib/pglogical_output/doc/protocol.txt
create mode 100644 contrib/pglogical_output/expected/basic_json.out
create mode 100644 contrib/pglogical_output/expected/basic_json_1.out
create mode 100644 contrib/pglogical_output/expected/basic_native.out
create mode 100644 contrib/pglogical_output/expected/cleanup.out
create mode 100644 contrib/pglogical_output/expected/encoding_json.out
create mode 100644 contrib/pglogical_output/expected/hooks_json.out
create mode 100644 contrib/pglogical_output/expected/hooks_json_1.out
create mode 100644 contrib/pglogical_output/expected/hooks_native.out
create mode 100644 contrib/pglogical_output/expected/params_native.out
create mode 100644 contrib/pglogical_output/expected/params_native_1.out
create mode 100644 contrib/pglogical_output/expected/prep.out
create mode 100644 contrib/pglogical_output/pglogical_config.c
create mode 100644 contrib/pglogical_output/pglogical_config.h
create mode 100644 contrib/pglogical_output/pglogical_hooks.c
create mode 100644 contrib/pglogical_output/pglogical_hooks.h
create mode 100644 contrib/pglogical_output/pglogical_output.c
create mode 100644 contrib/pglogical_output/pglogical_output.h
create mode 100644 contrib/pglogical_output/pglogical_output/README
create mode 100644 contrib/pglogical_output/pglogical_output/hooks.h
create mode 100644 contrib/pglogical_output/pglogical_proto.c
create mode 100644 contrib/pglogical_output/pglogical_proto.h
create mode 100644 contrib/pglogical_output/pglogical_proto_json.c
create mode 100644 contrib/pglogical_output/pglogical_proto_json.h
create mode 100644 contrib/pglogical_output/pglogical_proto_native.c
create mode 100644 contrib/pglogical_output/pglogical_proto_native.h
create mode 100644 contrib/pglogical_output/regression.conf
create mode 100644 contrib/pglogical_output/sql/basic_json.sql
create mode 100644 contrib/pglogical_output/sql/basic_native.sql
create mode 100644 contrib/pglogical_output/sql/basic_setup.sql
create mode 100644 contrib/pglogical_output/sql/basic_teardown.sql
create mode 100644 contrib/pglogical_output/sql/cleanup.sql
create mode 100644 contrib/pglogical_output/sql/encoding_json.sql
create mode 100644 contrib/pglogical_output/sql/hooks_json.sql
create mode 100644 contrib/pglogical_output/sql/hooks_native.sql
create mode 100644 contrib/pglogical_output/sql/hooks_setup.sql
create mode 100644 contrib/pglogical_output/sql/hooks_teardown.sql
create mode 100644 contrib/pglogical_output/sql/params_native.sql
create mode 100644 contrib/pglogical_output/sql/prep.sql
create mode 100644 contrib/pglogical_output_plhooks/.gitignore
create mode 100644 contrib/pglogical_output_plhooks/Makefile
create mode 100644 contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
diff --git a/contrib/Makefile b/contrib/Makefile
index bd251f6..028fd9a 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -35,6 +35,8 @@ SUBDIRS = \
pg_stat_statements \
pg_trgm \
pgcrypto \
+ pglogical_output \
+ pglogical_output_plhooks \
pgrowlocks \
pgstattuple \
postgres_fdw \
diff --git a/contrib/pglogical_output/.gitignore b/contrib/pglogical_output/.gitignore
new file mode 100644
index 0000000..2322e13
--- /dev/null
+++ b/contrib/pglogical_output/.gitignore
@@ -0,0 +1,6 @@
+pglogical_output.so
+results/
+regression.diffs
+tmp_install/
+tmp_check/
+log/
diff --git a/contrib/pglogical_output/Makefile b/contrib/pglogical_output/Makefile
new file mode 100644
index 0000000..bc95140
--- /dev/null
+++ b/contrib/pglogical_output/Makefile
@@ -0,0 +1,27 @@
+MODULE_big = pglogical_output
+PGFILEDESC = "pglogical_output - logical replication output plugin"
+
+OBJS = pglogical_output.o pglogical_hooks.o pglogical_config.o \
+ pglogical_proto.o pglogical_proto_native.o \
+ pglogical_proto_json.o
+
+REGRESS = prep params_native basic_native hooks_native basic_json hooks_json encoding_json cleanup
+
+
+subdir = contrib/pglogical_output
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# 'make installcheck' disabled when building in-tree because these tests
+# require "wal_level=logical", which typical installcheck users do not have
+# (e.g. buildfarm clients).
+installcheck:
+ ;
+
+EXTRA_INSTALL += contrib/pglogical_output_plhooks
+EXTRA_REGRESS_OPTS += --temp-config=./regression.conf
+
+install: all
+ $(MKDIR_P) '$(DESTDIR)$(includedir)'/pglogical_output
+ $(INSTALL_DATA) pglogical_output/hooks.h '$(DESTDIR)$(includedir)'/pglogical_output
diff --git a/contrib/pglogical_output/README.md b/contrib/pglogical_output/README.md
new file mode 100644
index 0000000..23ff9c2
--- /dev/null
+++ b/contrib/pglogical_output/README.md
@@ -0,0 +1,622 @@
+# `pglogical` Output Plugin
+
+This is the [logical decoding](http://www.postgresql.org/docs/current/static/logicaldecoding.html)
+[output plugin](http://www.postgresql.org/docs/current/static/logicaldecoding-output-plugin.html)
+for `pglogical`. Its purpose is to extract a change stream from a PostgreSQL
+database and send it to a client over a network connection using a
+well-defined, efficient protocol that multiple different applications can
+consume.
+
+The primary purpose of `pglogical_output` is to supply data to logical
+streaming replication solutions, but any application can potentially use its
+data stream. The output stream is designed to be compact and fast to decode,
+and the plugin supports upstream filtering of data so that only the required
+information is sent.
+
+Only one database is replicated, rather than the whole PostgreSQL install. A
+subset of that database may be selected for replication, currently based on
+table and on replication origin. Filtering by a WHERE clause can be supported
+easily in future.
+
+No triggers are required to collect the change stream and no external ticker or
+other daemon is required. It's accumulated using
+[replication slots](http://www.postgresql.org/docs/current/static/logicaldecoding-explanation.html#AEN66446),
+as supported in PostgreSQL 9.4 or newer, and sent on top of the
+[PostgreSQL streaming replication protocol](http://www.postgresql.org/docs/current/static/protocol-replication.html).
+
+Unlike block-level ("physical") streaming replication, the change stream from
+the `pglogical` output plugin is compatible across different PostgreSQL
+versions and can even be consumed by non-PostgreSQL clients.
+
+Because logical decoding is used, only the changed rows are sent on the wire.
+There's no index change data, no vacuum activity, etc transmitted.
+
+The use of a replication slot means that the change stream is reliable and
+crash-safe. If the client disconnects or crashes it can reconnect and resume
+replay from the last message that client processed. Server-side changes that
+occur while the client is disconnected are accumulated in the queue to be sent
+when the client reconnects. This reliability also means that server-side
+resources are consumed whether or not a client is connected.
+
+# Why another output plugin?
+
+See [`DESIGN.md`](DESIGN.md) for a discussion of why using one of the existing
+generic logical decoding output plugins like `wal2json` to drive a logical
+replication downstream isn't ideal. It's mostly about speed.
+
+# Architecture and high level interaction
+
+The output plugin is loaded by a PostgreSQL walsender process when a client
+connects to PostgreSQL using the PostgreSQL wire protocol with connection
+option `replication=database`, then uses
+[the `CREATE_REPLICATION_SLOT ... LOGICAL ...` or `START_REPLICATION SLOT ... LOGICAL ...` commands](http://www.postgresql.org/docs/current/static/logicaldecoding-walsender.html) to start streaming changes. (It can also be used via
+[SQL level functions](http://www.postgresql.org/docs/current/static/logicaldecoding-sql.html)
+over a non-replication connection, but this is mainly for debugging purposes).
+
+The client supplies parameters to the `START_REPLICATION SLOT ... LOGICAL ...`
+command to specify the version of the `pglogical` protocol it supports,
+whether it wants binary format, etc.
+
+The output plugin processes the connection parameters and the connection enters
+streaming replication protocol mode, sometimes called "COPY BOTH" mode because
+it's based on the protocol used for the `COPY` command. PostgreSQL then calls
+functions in this plugin to send it a stream of transactions to decode and
+translate into network messages. This stream of changes continues until the
+client disconnects.
+
+The only client-to-server interaction after startup is the sending of periodic
+feedback messages that allow the replication slot to discard no-longer-needed
+change history. The client *must* send feedback, otherwise `pg_xlog` on the
+server will eventually fill up and the server will stop working.
+
+
+# Usage
+
+The overall flow of client/server interaction is:
+
+* Client makes PostgreSQL fe/be protocol connection to server
+ * Connection options must include `replication=database` and `dbname=[...]` parameters
+ * The PostgreSQL client library can be `libpq` or anything else that supports the replication sub-protocol
+ * The same mechanisms are used for authentication and protocol encryption as for a normal non-replication connection
+* [Client issues `IDENTIFY_SYSTEM`
+ * Server responds with a single row containing system identity info
+* Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'` if it's setting up for the first time
+ * Server responds with success info and a snapshot identifier
+ * Client may at this point use the snapshot identifier on other connections while leaving this one idle
+* Client issues `START_REPLICATION SLOT slotname LOGICAL 0/0 (...options...)` to start streaming, which loops:
+ * Server emits `pglogical` message block encapsulated in a replication protocol `CopyData` message
+ * Client receives and unwraps message, then decodes the `pglogical` message block
+ * Client intermittently sends a standby status update message to server to confirm replay
+* ... until client sends a graceful connection termination message on the fe/be protocol level or the connection is broken
+
+ The details of `IDENTIFY_SYSTEM`, `CREATE_REPLICATION_SLOT` and `START_REPLICATION` are discussed in the [replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html) and will not be repeated here.
+
+## Make a replication connection
+
+To use the `pglogical` plugin you must first establish a PostgreSQL FE/BE
+protocol connection using the client library of your choice, passing
+`replication=database` as one of the connection parameters. `database` is a
+literal string and is not replaced with the database name; instead the database
+name is passed separately in the usual `dbname` parameter. Note that
+`replication` is not a GUC (configuration parameter) and may not be passed in
+the `options` parameter on the connection, it's a top-level parameter like
+`user` or `dbname`.
+
+Example connection string for `libpq`:
+
+ 'user=postgres replication=database sslmode=verify-full dbname=mydb'
+
+The plug-in name to pass on logical slot creation is `'pglogical'`.
+
+Details are in the replication protocol docs.
+
+## Get system identity
+
+If required you can use the `IDENTIFY_SYSTEM` command, which reports system
+information:
+
+ systemid | timeline | xlogpos | dbname | dboid
+ ---------------------+----------+-----------+--------+-------
+ 6153224364663410513 | 1 | 0/C429C48 | testd | 16385
+ (1 row)
+
+Details are in the replication protocol docs.
+
+## Create the slot if required
+
+If your application creates its own slots on first use and hasn't previously
+connected to this database on this system you'll need to create a replication
+slot. This keeps track of the client's replay state even while it's disconnected.
+
+The slot name may be anything your application wants up to a limit of 63
+characters in length. It's strongly advised that the slot name clearly identify
+the application and the host it runs on.
+
+Pass `pglogical` as the plugin name.
+
+e.g.
+
+ CREATE_REPLICATION_SLOT "reporting_host_42" LOGICAL "pglogical";
+
+`CREATE_REPLICATION_SLOT` returns a snapshot identifier that may be used with
+[`SET TRANSACTION SNAPSHOT`](http://www.postgresql.org/docs/current/static/sql-set-transaction.html)
+to see the database's state as of the moment of the slot's creation. The first
+change streamed from the slot will be the change immediately after this
+snapshot was taken. The snapshot is useful when cloning the initial state of a
+database being replicted. Applications that want to see the change stream
+going forward, but don't care about the initial state, can ignore this. The
+snapshot is only valid as long as the connection that issued the
+`CREATE_REPLICATION_SLOT` remains open and has not run another command.
+
+## Send replication parameters
+
+The client now sends:
+
+ START_REPLICATION SLOT "the_slot_name" LOGICAL (
+ 'Expected_encoding', 'UTF8',
+ 'Max_proto_major_version', '1',
+ 'Min_proto_major_version', '1',
+ ...moreparams...
+ );
+
+to start replication.
+
+The parameters are very important for ensuring that the plugin accepts
+the replication request and streams changes in the expected form. `pglogical`
+parameters are discussed in the separate `pglogical` protocol documentation.
+
+## Process the startup message
+
+`pglogical`'s output plugin will send a `CopyData` message containing its
+startup message as the first protocol message. This message contains a
+set of key/value entries describing the capabilities of the upstream output
+plugin, its version and the Pg version, the tuple format options selected,
+etc.
+
+The downstream client may choose to cleanly close the connection and disconnect
+at this point if it doesn't like the reply. It might then inform the user
+or reconnect with different parameters based on what it learned from the
+first connection's startup message.
+
+## Consume the change stream
+
+`pglogical`'s output plugin now sends a continuous series of `CopyData`
+protocol messages, each of which encapsulates a `pglogical` protocol message
+as documented in the separate protocol docs.
+
+These messages provide information about transaction boundaries, changed
+rows, etc.
+
+The stream continues until the client disconnects, the upstream server is
+restarted, the upstream walsender is terminated by admin action, there's
+a network issue, or the connection is otherwise broken.
+
+The client should send periodic feedback messages to the server to acknowledge
+that it's replayed to a given point and let the server release the resources
+it's holding in case that change stream has to be replayed again. See
+["Hot standby feedback message" in the replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html)
+for details.
+
+## Disconnect gracefully
+
+Disconnection works just like any normal client; you use your client library's
+usual method for closing the connection. No special action is required before
+disconnection, though it's usually a good idea to send a final standby status
+message just before you disconnect.
+
+# Tests
+
+The `pg_regress` tests check invalid parameter handling and basic
+functionality. They're intended for use by the buildfarm using an in-tree
+`make check`, but may also be run with an out-of-tree PGXS build against an
+existing PostgreSQL install using `make USE_PGXS=1 clean installcheck`.
+
+The tests may fail on installations that are not utf-8 encoded because the
+payloads of the binary protocol output will have text in different encodings,
+which aren't visible to psql as text to be decoded. Avoiding anything except
+7-bit ascii in the tests *should* prevent the problem.
+
+# Changeset forwarding
+
+It's possible to use `pglogical_output` to cascade replication between multiple
+PostgreSQL servers, in combination with an appropriate client to apply the
+changes to the downstreams.
+
+There are three forwarding modes:
+
+* Forward everything. Transactions are replicated whether they were made directly
+ on the immediate upstream or some other node upstream of it. All rows from all
+ transactions are sent.
+
+ Selected by setting `forward_changesets` to true (default) and not setting a
+ row or transaction filter hook.
+
+* No forwarding. Only transactions applied immediately on the upstream node are
+ forwarded. Transactions with any non-local origin are skipped. All rows from
+ locally originated transactions are sent.
+
+ Selected by setting `forward_changesets` to false. Remember to confirm by
+ checking the startup reply message.
+
+* Filtered forwarding. Transactions are replicated unless a client-supplied
+ transaction filter hook says to skip this transaction. Row changes are
+ replicated unless the client-supplied row filter hook (if provided) says to
+ skip that row.
+
+ Selected by setting `forward_changesets` to `true` and installing a
+ transaction and/or row filter hook (see "hooks").
+
+If the upstream server is 9.5 or newer and `forward_changesets` is enabled, the
+server will enable changeset origin information. It will set
+`forward_changeset_origins` to true in the startup reply message to indicate
+this. It will then send changeset origin messages after the `BEGIN` for each
+transaction, per the protocol documentation. Origin messages are omitted for
+transactions originating directly on the immediate upstream to save bandwidth.
+If `forward_changeset_origins` is true then transactions without an origin are
+always from the immediate upstream that’s running the decoding plugin.
+
+Note that changeset forwarding may be forced to on if not requested by some
+servers, so the client _should_ check the forward_changesets and
+`forward_changeset_origins` params in the startup reply message.
+
+Clients may use this facility to form arbitrarily complex topologies when
+combined with hooks to determine which transactions are forwarded. An obvious
+case is bi-directional (mutual) replication.
+
+# Selective replication
+
+By specifying a row filter hook it's possible to filter the replication stream
+server-side so that only a subset of changes is replicated.
+
+
+# Hooks
+
+`pglogical_output` exposes a number of extension points where applications can
+modify or override its behaviour.
+
+All hooks are called in their own memory context, which lasts for the duration
+of the logical decoding session. They may switch to longer lived contexts if
+needed, but are then responsible for their own cleanup.
+
+## Hook setup function
+
+The downstream must specify the fully-qualified name of a SQL-callable function
+on the server as the value of the `hooks.setup_function` client parameter.
+The SQL signature of this function is
+
+ CREATE OR REPLACE FUNCTION funcname(hooks internal, memory_context internal)
+ RETURNS void STABLE
+ LANGUAGE c AS 'MODULE_PATHNAME';
+
+Permissions are checked. This function must be callable by the user that the
+output plugin is running as. The function name *must* be schema-qualified and is
+parsed like any other qualified identifier.
+
+The function receives a pointer to a newly allocated structure of hook function
+pointers to populate as its first argument. The function must not free the
+argument.
+
+If the hooks need a private data area to store information across calls, the
+setup function should get the `MemoryContext` pointer from the 2nd argument,
+then `MemoryContextAlloc` a struct for the data in that memory context and
+store the pointer to it in `hooks->hooks_private_data`. This will then be
+accessible on future calls to hook functions. It need not be manually freed, as
+the memory context used for logical decoding will free it when it's freed.
+Don't put anything in it that needs manual cleanup.
+
+Each hook has its own C signature (defined below) and the pointers must be
+directly to the functions. Hooks that the client does not wish to set must be
+left null.
+
+An example is provided in `contrib/pglogical_output_plhooks` and the argument
+structs are defined in `pglogical_output/hooks.h`, which is installed into the
+PostgreSQL source tree when the extension is installed.
+
+Each hook that is enabled results in a new startup parameter being emitted in
+the startup reply message. Clients must check for these and must not assume a
+hook was successfully activated because no error is seen.
+
+Hook functions are called in the context of the backend doing logical decoding.
+Except for the startup hook, hooks see the catalog state as it was at the time
+the transaction or row change being examined was made. Access to to non-catalog
+tables is unsafe unless they have the `user_catalog_table` reloption set.
+
+## Startup hook
+
+The startup hook is called when logical decoding starts.
+
+This hook can inspect the parameters passed by the client to the output
+plugin as in_params. These parameters *must not* be modified.
+
+It can add new parameters to the set to be returned to the client in the
+startup parameters message, by appending to List out_params, which is
+initially NIL. Each element must be a `DefElem` with the param name
+as the `defname` and a `String` value as the arg, as created with
+`makeDefElem(...)`. It and its contents must be allocated in the
+logical decoding memory context.
+
+For walsender based decoding the startup hook is called only once, and
+cleanup might not be called at the end of the session.
+
+Multiple decoding sessions, and thus multiple startup hook calls, may happen
+in a session if the SQL interface for logical decoding is being used. In
+that case it's guaranteed that the cleanup hook will be called between each
+startup.
+
+When successfully enabled, the output parameter `hooks.startup_hook_enabled` is
+set to true in the startup reply message.
+
+Unlike the other hooks, this hook sees a snapshot of the database's current
+state, not a time-traveled catalog state. It is safe to access all tables from
+this hook.
+
+## Transaction filter hook
+
+The transaction filter hook can exclude entire transactions from being decoded
+and replicated based on the node they originated from.
+
+It is passed a `const TxFilterHookArgs *` containing:
+
+* The hook argument supplied by the client, if any
+* The `RepOriginId` that this transaction originated from
+
+and must return boolean, where true retains the transaction for sending to the
+client and false discards it. (Note that this is the reverse sense of the low
+level logical decoding transaction filter hook).
+
+The hook function must *not* free the argument struct or modify its contents.
+
+Note that individual changes within a transaction may have different origins to
+the transaction as a whole; see "Origin filtering" for more details. If a
+transaction is filtered out, all changes are filtered out even if their origins
+differ from that of the transaction as a whole.
+
+When successfully enabled, the output parameter
+`hooks.transaction_filter_enabled` is set to true in the startup reply message.
+
+## Row filter hook
+
+The row filter hook is called for each row. It is passed information about the
+table, the transaction origin, and the row origin.
+
+It is passed a `const RowFilterHookArgs*` containing:
+
+* The hook argument supplied by the client, if any
+* The `Relation` the change affects
+* The change type - 'I'nsert, 'U'pdate or 'D'elete
+
+It can return true to retain this row change, sending it to the client, or
+false to discard it.
+
+The function *must not* free the argument struct or modify its contents.
+
+Note that it is more efficient to exclude whole transactions with the
+transaction filter hook rather than filtering out individual rows.
+
+When successfully enabled, the output parameter
+`hooks.row_filter_enabled` is set to true in the startup reply message.
+
+## Shutdown hook
+
+The shutdown hook is called when a decoding session ends. You can't rely on
+this hook being invoked reliably, since a replication-protocol walsender-based
+session might just terminate. It's mostly useful for cleanup to handle repeated
+invocations under the SQL interface to logical decoding.
+
+You don't need a hook to free memory you allocated, unless you explicitly
+switched to a longer lived memory context like TopMemoryContext. Memory allocated
+in the hook context will be automatically when the decoding session shuts down.
+
+## Writing hooks in procedural languages
+
+You can write hooks in PL/PgSQL, etc, too, via the `pglogical_output_plhooks`
+adapter extension in `contrib`. They won't perform very well though.
+
+# Limitations
+
+The advantages of logical decoding in general and `pglogical_output` in
+particular are discussed above. There are also some limitations that apply to
+`pglogical_output`, and to Pg's logical decoding in general.
+
+(TODO: move much of this to the main logical decoding docs)
+
+Notably:
+
+## Doesn't replicate DDL
+
+Logical decoding doesn't decode catalog changes directly. So the plugin can't
+just send a `CREATE TABLE` statement when a new table is added.
+
+If the data being decoded is being applied to another PostgreSQL database then
+its table definitions must be kept in sync via some means external to the logical
+decoding plugin its self, such as:
+
+* Event triggers using DDL deparse to capture DDL changes as they happen and write them to a table to be replicated and applied on the other end; or
+* doing DDL management via tools that synchronise DDL on all nodes
+
+## Doesn't replicate global objects/shared catalog changes
+
+PostgreSQL has a number of object types that exist across all databases, stored
+in *shared catalogs*. These include:
+
+* Roles (users/groups)
+* Security labels on users and databases
+
+Such objects cannot be replicated by `pglogical_output`. They're managed with DDL that
+can't be captured within a single database and isn't decoded anyway.
+
+DDL for global object changes must be synchronized via some external means.
+
+## Mostly one-way communication
+
+Per the protocol documentation, the downstream can't send anything except
+replay progress messages to the upstream after replication begins, and can't
+re-initialise replication without a disconnect.
+
+To achieve downstream-to-upstream communication, clients can use a regular
+libpq connection to the upstream then write to tables or call functions.
+Alternately, a separate replication connection in the opposite direction can be
+created by the application to carry information from downstream to upstream.
+
+See "Protocol flow" in the protocol documentation for more information.
+
+## Physical replica failover
+
+Logical decoding cannot follow a physical replication failover because
+replication slot state is not replicated to physical replicas. If you fail over
+to a streaming replica you have to manually reconnect your logical replication
+clients, creating new slots, etc. This is a core PostgreSQL limitation.
+
+Also, there's no built-in way to guarantee that the logical replication slot
+from the failed master hasn't replayed further than the physical streaming
+replica you failed over to. You could receive changes on your logical decoding
+stream from the old master that never made it to the physical streaming
+replica. This is true (albeit very unlikely) *even if the physical streaming
+replica is synchronous* because PostgreSQL sends the replication data anyway,
+then just delays the commit's visibility on the master. Support for strictly
+ordered standbys would be required in PostgreSQL to avoid this.
+
+To achieve failover with logical replication you cannot mix in physical
+standbys. The logical replication client has to take responsibility for
+maintaining slots on logical replicas intended as failover candidates
+and for ensuring that the furthest-ahead replica is promoted if there is
+more than one.
+
+## Can only replicate complete transactions
+
+Logical decoding can only replicate a transaction after it has committed. This
+usefully skips replication of rolled back transactions, but it also means that
+very large transactions must be completed upstream before they can begin on the
+downstream, adding to replication latency.
+
+## Replicates only one transaction at a time
+
+Logical decoding serializes transactions in commit order, so pglogical_output
+cannot replay interleaved concurrent transactions. This can lead to high latencies
+when big transactions are being replayed, since smaller transactions get queued
+up behind them.
+
+## Unique index required for inserts or updates
+
+To replicate `INSERT`s or `UPDATE`s it is necessary to have a `PRIMARY KEY`
+or a (non-partial, columns-only) `UNIQUE` index on the table, so the table
+has a `REPLICA IDENTITY`. Without that `pglogical_output` doesn't know what
+old key to send to allow the receiver to tell which tuple is being updated.
+
+## UNLOGGED tables aren't replicated
+
+Because `UNLOGGED` tables aren't written to WAL, they aren't replicated by
+logical or physical replication. You can only replicate `UNLOGGED` tables
+with trigger-based solutions.
+
+## Unchanged fields are often sent in `UPDATE`
+
+Because there's no tracking of dirty/clean fields when a tuple is updated,
+logical decoding can't tell if a given field was changed by an update.
+Unchanged fields can only by identified and omitted if they're a variable
+length TOASTable type and are big enough to get stored out-of-line in
+a TOAST table.
+
+# Troubleshooting and debugging
+
+## Non-destructively previewing pending data on a slot
+
+Using the json mode of `pglogical_output` you can examine pending transactions
+on a slot without consuming them, so they are still delivered to the usual
+client application that created/owns this slot. This is best done using the SQL
+interface to logical decoding, since it gives you finer control than using
+`pg_recvlogical`.
+
+You can only peek at a slot while there is no other client connected to that
+slot.
+
+Use `pg_logical_slot_peek_changes` to examine the change stream without
+destructively consuming changes. This is extremely helpful when trying to
+determine why an error occurs in a downstream, since you can examine a
+json-ified representation of the xact. It's necessary to supply a minimal
+set of required parameters to the output plugin.
+
+e.g. given setup:
+
+ CREATE TABLE discard_test(blah text);
+ SELECT 'init' FROM pg_create_logical_replication_slot('demo_slot', 'pglogical_output');
+ INSERT INTO discard_test(blah) VALUES('one');
+ INSERT INTO discard_test(blah) VALUES('two1'),('two2'),('two3');
+ INSERT INTO discard_test(blah) VALUES('three1'),('three2');
+
+you can peek at the change stream with:
+
+ SELECT location, xid, data
+ FROM pg_logical_slot_peek_changes('demo_slot', NULL, NULL,
+ 'min_proto_version', '1', 'max_proto_version', '1',
+ 'startup_params_format', '1', 'proto_format', 'json');
+
+The two `NULL`s mean you don't want to stop decoding after any particular
+LSN or any particular number of changes. Decoding will stop when there's nothing
+left to decode or you cancel the query.
+
+This will emit a key/value startup message then change data rows like:
+
+ location | xid | data
+ 0/4E8AAF0 | 5562 | {"action":"B", has_catalog_changes:"f", xid:"5562", first_lsn:"0/4E8AAF0", commit_time:"2015-11-13 14:26:21.404425+08"}
+ 0/4E8AAF0 | 5562 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"one"}}
+ 0/4E8AB70 | 5562 | {"action":"C", final_lsn:"0/4E8AB30", end_lsn:"0/4E8AB70"}
+ 0/4E8ABA8 | 5563 | {"action":"B", has_catalog_changes:"f", xid:"5563", first_lsn:"0/4E8ABA8", commit_time:"2015-11-13 14:26:32.015611+08"}
+ 0/4E8ABA8 | 5563 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"two1"}}
+ 0/4E8ABE8 | 5563 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"two2"}}
+ 0/4E8AC28 | 5563 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"two3"}}
+ 0/4E8ACA8 | 5563 | {"action":"C", final_lsn:"0/4E8AC68", end_lsn:"0/4E8ACA8"}
+ ....
+
+The output is the LSN (log sequence number) associated with a change, the top
+level transaction ID that performed the change, and the change data as json.
+
+You can see the transaction boundaries by xid changes and by the "B"egin and
+"C"ommit messages, and you can see the individual row "I"nserts. Replication
+origins, commit timestamps, etc will be shown if known.
+
+See http://www.postgresql.org/docs/current/static/functions-admin.html for
+information on the peek functions.
+
+If you want the binary format you can get that with
+`pg_logical_slot_peek_binary_changes` and the `native` protocol, but that's
+generally much less useful.
+
+# Manually discarding a change from a slot
+
+Sometimes it's desirable to manually purge one or more changes from a
+replication slot. This is usually an error recovery step when problems arise
+with the downstream code that's replaying from the slot.
+
+You can use the peek functions to determine the point in the stream you want to
+discard up to, as identifed by LSN (log sequence number). See
+"non-destructively previewing pending data on a slot" above for details.
+
+You can't control the point you start discarding from, it's always from the
+current stream position up to a point you specify. If the peek shows that
+there's data you still want to retain you must make sure that the downstream
+replays up to the point you want to keep changes and sends replay confirmation.
+In other words there's no way to cut a sequence of changes out of the middle of
+the pending change stream.
+
+Once you've peeked the stream and know the LSN you want to discard up to, you
+can use `pg_logical_slot_peek_changes`, specifying an `upto_lsn`, to consume
+changes from the slot up to but not including that point, i.e. that will be the
+point at which replay resumes.
+
+For example, if you wanted to discard the first transaction in the example
+from the section above, i.e. discard xact 5562 and start decoding at xact
+5563 from its' BEGIN lsn `0/4E8ABA8`, you'd run:
+
+ SELECT location, xid, data
+ FROM pg_logical_slot_get_changes('demo_slot', '0/4E8ABA8', NULL,
+ 'min_proto_version', '1', 'max_proto_version', '1',
+ 'startup_params_format', '1', 'proto_format', 'json');
+
+Note that `_get_changes` is used instead of `_peek_changes` and that
+the `upto_lsn` is `'0/4E8ABA8'` instead of `NULL`.
+
+
+
+
+
diff --git a/contrib/pglogical_output/doc/.gitignore b/contrib/pglogical_output/doc/.gitignore
new file mode 100644
index 0000000..2874bff
--- /dev/null
+++ b/contrib/pglogical_output/doc/.gitignore
@@ -0,0 +1 @@
+protocol.html
diff --git a/contrib/pglogical_output/doc/DESIGN.md b/contrib/pglogical_output/doc/DESIGN.md
new file mode 100644
index 0000000..05fb4d1
--- /dev/null
+++ b/contrib/pglogical_output/doc/DESIGN.md
@@ -0,0 +1,124 @@
+# Design decisions
+
+Explanations of why things are done the way they are.
+
+## Why does pglogical_output exist when there's wal2json etc?
+
+`pglogical_output` does plenty more than convert logical decoding change
+messages to a wire format and send them to the client.
+
+It handles format negotiations, sender-side filtering using pluggable hooks
+(and the associated plugin handling), etc. The protocol its self is also
+important, and incorporates elements like binary datum transfer that can't be
+easily or efficiently achieved with json.
+
+## Custom binary protocol
+
+Why do we have a custom binary protocol inside the walsender / copy both protocol,
+rather than using a json message representation?
+
+Speed and compactness. It's expensive to create json, with lots of allocations.
+It's expensive to decode it too. You can't represent raw binary in json, and must
+encode it, which adds considerable overhead for some data types. Using the
+obvious, easy to decode json representations also makes it difficult to do
+later enhancements planned for the protocol and decoder, like caching row
+metadata.
+
+The protocol implementation is fairly well encapsulated, so in future it should
+be possible to emit json instead for clients that request it. Right now that's
+not the priority as tools like wal2json already exist for that.
+
+## Column metadata
+
+The output plugin sends metadata for columsn - at minimum, the column names -
+before each row. It will soon be changed to send the data before each row from
+a new, different table, so that streams of inserts from COPY etc don't repeat
+the metadata each time. That's just a pending feature.
+
+The reason metadata must be sent is that the upstream and downstream table's
+attnos don't necessarily correspond. The column names might, and their ordering
+might even be the same, but any column drop or column type change will result
+in a dropped column on one side. So at the user level the tables look the same,
+but their attnos don't match, and if we rely on attno for replication we'll get
+the wrong data in the wrong columns. Not pretty.
+
+That could be avoided by requiring that the downstream table be strictly
+maintained by DDL replication, but:
+
+* We don't want to require DDL replication
+* That won't work with multiple upstreams feeding into a table
+* The initial table creation still won't be correct if the table has dropped
+ columns, unless we (ab)use `pg_dump`'s `--binary-upgrade` support to emit
+ tables with dropped columns, which we don't want to do.
+
+So despite the bandwidth cost, we need to send metadata.
+
+In future a client-negotiated cache is planned, so that clients can announce
+to the output plugin that they can cache metadata across change series, and
+metadata can only be sent when invalidated by relation changes or when a new
+relation is seen.
+
+Support for type metadata is penciled in to the protocol so that clients that
+don't have table definitions at all - like queueing engines - can decode the
+data. That'll also permit type validation sanity checking on the apply side
+with logical replication.
+
+## Hook entry point as a SQL function
+
+The hooks entry point is a SQL function that populates a passed `internal`
+struct with hook function pointers.
+
+The reason for this is that hooks are specified by a remote peer over the
+network. We can't just let the peer say "dlsym() this arbitrary function name
+and call it with these arguments" for fairly obvious security reasons. At bare
+minimum all replication using hooks would have to be superuser-only if we did
+that.
+
+The SQL entry point is only called once per decoding session and the rest of
+the calls are plain C function pointers.
+
+## The startup reply message
+
+The protocol design choices available to `pg_logical` are constrained by being
+contained in the copy-both protocol within the fe/be protocol, running as a
+logical decoding plugin. The plugin has no direct access to the network socket
+and can't send or receive messages whenever it wants, only under the control of
+the walsender and logical decoding framework.
+
+The only opportunity for the client to send data directly to the logical
+decoding plugin is in the `START_REPLICATION` parameters, and it can't send
+anything to the client before that point.
+
+This means there's no opportunity for a multi-way step negotiation between
+client and server. We have to do all the negotiation we're going to in a single
+exchange of messages - the setup parameters and then the replication start
+message. All the client can do if it doesn't like the offer the server makes is
+disconnect and try again with different parameters.
+
+That's what the startup message is for. It reports the plugin's capabilities
+and tells the client which requested options were honoured. This gives the
+client a chance to decide if it's happy with the output plugin's decision
+or if it wants to reconnect and try again with different options. Iterative
+negotiation, effectively.
+
+## Unrecognised parameters MUST be ignored by client and server
+
+To ensure upward and downward compatibility, the output plugin must ignore
+parameters set by the client if it doesn't recognise them, and the client
+must ignore parameters it doesn't recognise in the server's startup reply
+message.
+
+This ensures that older clients can talk to newer servers and vice versa.
+
+For this to work, the server must never enable new functionality such as
+protocol message types, row formats, etc without the client explicitly
+specifying via a startup parameter that it understands the new functionality.
+Everything must be negotiated.
+
+Similarly, a newer client talking to an older server may ask the server to
+enable functionality, but it can't assume the server will actually honour that
+request. It must check the server's startup reply message to see if the server
+confirmed that it enabled the requested functionality. It might choose to
+disconnect and report an error to the user if the server didn't do what it
+asked. This can be important, e.g. when a security-significant hook is
+specified.
diff --git a/contrib/pglogical_output/doc/protocol.txt b/contrib/pglogical_output/doc/protocol.txt
new file mode 100644
index 0000000..0ab41bf
--- /dev/null
+++ b/contrib/pglogical_output/doc/protocol.txt
@@ -0,0 +1,546 @@
+= Pg_logical protocol
+
+pglogical_output defines a libpq subprocotol for streaming tuples, metadata,
+etc, from the decoding plugin to receivers.
+
+This protocol is an inner layer in a stack:
+
+ * tcp or unix sockets
+ ** libpq protocol
+ *** libpq replication subprotocol (COPY BOTH etc)
+ **** pg_logical output plugin => consumer protocol
+
+so clients can simply use libpq's existing replication protocol support,
+directly or via their libpq-wrapper driver.
+
+This is a binary protocol intended for compact representation.
+
+`pglogical_output` also supports a json-based text protocol with json
+representations of the same changesets, supporting all the same hooks etc,
+intended mainly for tracing/debugging/diagnostics. That protocol is not
+discussed here.
+
+== ToC
+
+== Protocol flow
+
+The protocol flow is primarily from upstream walsender/decoding plugin to the
+downstream receiver.
+
+The only information the flows downstream-to-upstream is:
+
+ * The initial parameter list sent to `START_REPLICATION`; and
+ * replay progress messages
+
+We can accept an arbitrary list of params to `START_REPLICATION`. After
+that we have no general purpose channel for information to flow upstream. That
+means we can't do a multi-step negotiation/handshake for determining the
+replication options to use, binary protocol, etc.
+
+The main form of negotiation is the client getting a "take it or leave it" set
+of settings from the server in an initial startup message sent before any
+replication data (see below) and, if it doesn't like them, reconnecting with
+different startup options.
+
+Except for the negotiation via initial parameter list and then startup message
+the protocol flow is the same as any other walsender-based logical replication
+plugin. The data stream is sent in COPY BOTH mode as a series of CopyData
+messages encapsulating replication data, and ends when the client disconnects.
+There's no facility for ending the COPY BOTH mode and returning to the
+walsender command parser to issue new commands. This is a limiation of the
+walsender interface, not pglogical_output.
+
+== Protocol messages
+
+The individual protocol messages are discussed in the following sub-sections.
+Protocol flow and logic comes in the next major section.
+
+Absolutely all top-level protocol messages begin with a message type byte.
+While represented in code as a character, this is a signed byte with no
+associated encoding.
+
+Since the PostgreSQL libpq COPY protocol supplies a message length there’s no
+need for top-level protocol messages to embed a length in their header.
+
+=== BEGIN message
+
+A stream of rows starts with a `BEGIN` message. Rows may only be sent after a
+`BEGIN` and before a `COMMIT`.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**B**’ (0x42)
+|flags|uint8| * 0-3: Reserved, client _must_ ERROR if set and not recognised.
+|lsn|uint64|“final_lsn” in decoding context - currently it means lsn of commit
+|commit time|uint64|“commit_time” in decoding context
+|remote XID|uint32|“xid” in decoding context
+|===
+
+=== Forwarded transaction origin message
+
+The message after the `BEGIN` may be a _forwarded transaction origin_ message
+indicating what upstream node the transaction came from.
+
+Sent if the immediately prior message was a `BEGIN` message, the upstream
+transaction was forwarded from another node, and replication origin forwarding
+is enabled, i.e. `forward_changeset_origins` is `t` in the startup reply
+message.
+
+A "node" could be another host, another DB on the same host, or pretty much
+anything. Whatever origin name is found gets forwarded. The origin identifier
+is of arbitrary and application-defined format. Applications _should_ prefix
+their origin identifier with a fixed application name part, like `bdr_`,
+`myapp_`, etc. It is application-defined what an application does with
+forwarded transactions from other applications.
+
+An origin message with a zero-length origin name indicates that the origin
+could not be identified but was (probably) not the local node. It is
+client-defined what action is taken in this case.
+
+It is a protocol error to send/receive a forwarded transaction origin message
+at any time other than immediately after a `BEGIN` message.
+
+The origin identifier is typically closely related to replication slot names
+and replication origins’ names in an application system.
+
+For more detail see _Changeset Forwarding_ in the README.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**O**’ (0x4f)
+|flags|uint8| * 0-3: Reserved, application _must_ ERROR if set and not recognised
+|origin_lsn|uint64|Log sequence number (LSN, XLogRecPtr) of the transaction’s commit record on its origin node (as opposed to the forwarding node’s commit LSN, which is ‘lsn’ in the BEGIN message)
+|origin_identifier_length|uint8|Length in bytes of origin_identifier
+|origin_identifier|signed char[origin_identifier_length]|An origin identifier of arbitrary, upstream-application-defined structure. _Should_ be text in the same encoding as the upstream database. NULL-terminated. _Should_ be 7-bit ASCII.
+|===
+
+=== COMMIT message
+A stream of rows ends with a `COMMIT` message.
+
+There is no `ROLLBACK` message because aborted transactions are not sent by the
+upstream.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**C**’ (0x43)
+|Flags|uint8| * 0-3: Reserved, client _must_ ERROR if set and not recognised
+|Commit LSN|uint64|commit_lsn in decoding commit decode callback. This is the same value as in the BEGIN message, and marks the end of the transaction.
+|End LSN|uint64|end_lsn in decoding transaction context
+|Commit time|uint64|commit_time in decoding transaction context
+|===
+
+=== INSERT, UPDATE or DELETE message
+
+After a `BEGIN` or metadata message, the downstream should expect to receive
+zero or more row change messages, composed of an insert/update/delete message
+with zero or more tuple fields, each of which has one or more tuple field
+values.
+
+The row’s relidentifier _must_ match that of the most recently preceding
+metadata message. All consecutive row messages must currently have the same
+relidentifier. (_Later extensions to add metadata caching will relax these
+requirements for clients that advertise caching support; see the documentation
+on metadata messages for more detail_).
+
+It is an error to decode rows using metadata received after the row was
+received, or using metadata that is not the most recently received metadata
+revision that still predates the row. I.e. in the sequence M1, R1, R2, M2, R3,
+M4: R1 and R2 must be decoded using M1, and R3 must be decoded using M2. It is
+an error to use M4 to decode any of the rows, to use M1 to decode R3, or to use
+M2 to decode R1 and R2.
+
+Row messages _may not_ arrive except during a transaction as delimited by `BEGIN`
+and `COMMIT` messages. It is an error to receive a row message outside a
+transaction.
+
+Any unrecognised tuple type or tuple part type is an error on the downstream
+that must result in a client disconnect and error message. Downstreams are
+expected to negotiate compatibility, and upstreams must not add new tuple types
+or tuple field types without negotiation.
+
+The downstream reads rows until the next non-row message is received. There is
+no other end marker or any indication of how many rows to expect in a sequence.
+
+==== Row message header
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**I**’nsert (0x49), ‘**U**’pdate’ (0x55) or ‘**D**’elete (0x44)
+|flags|uint8|Row flags (reserved)
+|relidentifier|uint32|relidentifier that matches the table metadata message sent for this row.
+(_Not present in BDR, which sends nspname and relname instead_)
+|[tuple parts]|[composite]|
+|===
+
+One or more tuple-parts fields follow.
+
+==== Tuple fields
+
+|===
+|Tuple type|signed char|Identifies the kind of tuple being sent.
+
+|tupleformat|signed char|‘**T**’ (0x54)
+|natts|uint16|Number of fields sent in this tuple part.
+(_Present in BDR, but meaning significantly different here)_
+|[tuple field values]|[composite]|
+|===
+
+===== Tuple tupleformat compatibility
+
+Unrecognised _tupleformat_ kinds are a protocol error for the downstream.
+
+==== Tuple field value fields
+
+These message parts describe individual fields within a tuple.
+
+There are two kinds of tuple value fields, abbreviated and full. Which is being
+read is determined based on the first field, _kind_.
+
+Abbreviated tuple value fields are nothing but the message kind:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|kind|signed char| * ‘**n**’ull (0x6e) field
+|===
+
+Full tuple value fields have a length and datum:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|kind|signed char| * ‘**i**’nternal binary (0x62) field
+|length|int4|Only defined for kind = i\|b\|t
+|data|[length]|Data in a format defined by the table metadata and column _kind_.
+|===
+
+===== Tuple field values kind compatibility
+
+Unrecognised field _kind_ values are a protocol error for the downstream. The
+downstream may not continue processing the protocol stream after this
+point**.**
+
+The upstream may not send ‘**i**’nternal or ‘**b**’inary format values to the
+downstream without the downstream negotiating acceptance of such values. The
+downstream will also generally negotiate to receive type information to use to
+decode the values. See the section on startup parameters and the startup
+message for details.
+
+=== Table/row metadata messages
+
+Before sending changed rows for a relation, a metadata message for the relation
+must be sent so the downstream knows the namespace, table name, column names,
+optional column types, etc. A relidentifier field, an arbitrary numeric value
+unique for that relation on that upstream connection, maps the metadata to
+following rows.
+
+A client should not assume that relation metadata will be followed immediately
+(or at all) by rows, since future changes may lead to metadata messages being
+delivered at other times. Metadata messages may arrive during or between
+transactions.
+
+The upstream may not assume that the downstream retains more metadata than the
+one most recent table metadata message. This applies across all tables, so a
+client is permitted to discard metadata for table x when getting metadata for
+table y. The upstream must send a new metadata message before sending rows for
+a different table, even if that metadata was already sent in the same session
+or even same transaction. _This requirement will later be weakened by the
+addition of client metadata caching, which will be advertised to the upstream
+with an output plugin parameter._
+
+Columns in metadata messages are numbered from 0 to natts-1, reading
+consecutively from start to finish. The column numbers do not have to be a
+complete description of the columns in the upstream relation, so long as all
+columns that will later have row values sent are described. The upstream may
+choose to omit columns it doesn’t expect to send changes for in any given
+series of rows. Column numbers are not necessarily stable across different sets
+of metadata for the same table, even if the table hasn’t changed structurally.
+
+A metadata message may not be used to decode rows received before that metadata
+message.
+
+==== Table metadata header
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**R**’ (0x52)
+|flags|uint8| * 0-6: Reserved, client _must_ ERROR if set and not recognised.
+|relidentifier|uint32|Arbitrary relation id, unique for this upstream. In practice this will probably be the upstream table’s oid, but the downstream can’t assume anything.
+|nspnamelength|uint8|Length of namespace name
+|nspname|signed char[nspnamelength]|Relation namespace (null terminated)
+|relnamelength|uint8|Length of relation name
+|relname|char[relname]|Relation name (null terminated)
+|attrs block|signed char|Literal: ‘**A**’ (0x41)
+|natts|uint16|number of attributes
+|[fields]|[composite]|Sequence of ‘natts’ column metadata blocks, each of which begins with a column delimiter followed by zero or more column metadata blocks, each with the same column metadata block header.
+
+This chunked format is used so that new metadata messages can be added without breaking existing clients.
+|===
+
+==== Column delimiter
+
+Each column’s metadata begins with a column metadata header. This comes
+immediately after the natts field in the table metadata header or after the
+last metadata block in the prior column.
+
+It has the same char header as all the others, and the flags field is the same
+size as the length field in other blocks, so it’s safe to read this as a column
+metadata block header.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|blocktype|signed char|‘**C**’ (0x43) - column
+|flags|uint8|Column info flags
+|===
+
+==== Column metadata block header
+
+All column metadata blocks share the same header, which is the same length as a
+column delimiter:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|blocktype|signed char|Identifies the kind of metadata block that follows.
+|blockbodylength|uint16|Length of block in bytes, excluding blocktype char and length field.
+|===
+
+==== Column name block
+
+This block just carries the name of the column, nothing more. It begins with a
+column metadata block, and the rest of the message is the column name.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|[column metadata block header]|[composite]|blocktype = ‘**N**’ (0x4e)
+|colname|char[blockbodylength]|Column name.
+|===
+
+
+==== Column type block
+
+T.B.D.
+
+Not defined in first protocol revision.
+
+Likely to send a type identifier (probably the upstream oid) as a reference to
+a “type info” protocol message to be delivered before. Then we can cache the
+type descriptions and avoid repeating long schemas and names, just using the
+oids.
+
+Needs to have room to handle:
+
+ * built-in core types
+ * extension types (ext version may vary)
+ * enum types (CREATE TYPE … AS ENUM)
+ * range types (CREATE TYPE … AS RANGE)
+ * composite types (CREATE TYPE … AS (...))
+ * custom types (CREATE TYPE ( input = x_in, output = x_out ))
+
+… some of which can be nested
+
+== Startup message
+
+After processing output plugin arguments, the upstream output plugin must send
+a startup message as its first message on the wire. It is a trivial header
+followed by alternating key and value strings represented as null-terminated
+unsigned char strings.
+
+This message specifies the capabilities the output plugin enabled and describes
+the upstream server and plugin. This may change how the client decodes the data
+stream, and/or permit the client to disconnect and report an error to the user
+if the result isn’t acceptable.
+
+If replication is rejected because the client is incompatible or the server is
+unable to satisfy required options, the startup message may be followed by a
+libpq protocol FATAL message that terminates the session. See “Startup errors”
+below.
+
+The parameter names and values are sent as alternating key/value pairs as
+null-terminated strings, e.g.
+
++“key1\0parameter1\0key2\0value2\0”+
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|‘**S**’ (0x53) - startup
+|Startup message version|uint8|Value is always “1”.
+|(parameters)|null-terminated key/value pairs|See table below for parameter definitions.
+|===
+
+=== Startup message parameters
+
+Since all parameter values are sent as strings, the value types given below specify what the value must be reasonably interpretable as.
+
+|===
+|*Key name*|*Value type*|*Description*
+
+|max_proto_version|integer|Newest version of the protocol supported by output plugin.
+|min_proto_version|integer|Oldest protocol version supported by server.
+|proto_format|text|Protocol format requested. native (documented here) or json. Default is native.
+|coltypes|boolean|Column types will be sent in table metadata.
+|pg_version_num|integer|PostgreSQL server_version_num of server, if it’s PostgreSQL. e.g. 090400
+|pg_version|string|PostgreSQL server_version of server, if it’s PostgreSQL.
+|pg_catversion|uint32|Version of the PostgreSQL system catalogs on the upstream server, if it’s PostgreSQL.
+|binary|_set of parameters, specified separately_|See “_the __‘binary’__ parameters_” below, and “_Parameters relating to exchange of binary values_”
+|database_encoding|string|The native text encoding of the database the plugin is running in
+|encoding|string|Field values for textual data will be in this encoding in native protocol text, binary or internal representation. For the native protocol this is currently always the same as `database_encoding`. For text-mode json protocol this is always the same as `client_encoding`.
+|forward_changesets|bool|Specifies that all transactions, not just those originating on the upstream, will be forwarded. See “_Changeset forwarding_”.
+|forward_changeset_origins|bool|Tells the client that the server will send changeset origin information. Independent of forward_changesets. See “_Changeset forwarding_” for details.
+|no_txinfo|bool|Requests that variable transaction info such as XIDs, LSNs, and timestamps be omitted from output. Mainly for tests. Currently ignored for protos other than json.
+|===
+
+
+The ‘binary’ parameter set:
+==
+|===
+|*Key name*|*Value type*|*Description*
+
+|binary.internal_basetypes|boolean|If true, PostgreSQL internal binary representations for row field data may be used for some or all row fields, if here the type is appropriate and the binary compatibility parameters of upstream and downstream match. See binary.want_internal_basetypes in the output plugin parameters for details.
+
+May only be true if _binary.want_internal_basetypes_ was set to true by the client in the parameters and the client’s accepted binary format matches that of the server.
+|binary.binary_basetypes|boolean|If true, external binary format (send/recv format) may be used for some or all row field data where the field type is a built-in base type whose send/recv format is compatible with binary.binary_pg_version .
+
+May only be set if _binary.want_binary_basetypes_ was set to true by the client in the parameters and the client’s accepted send/recv format matches that of the server.
+|binary.binary_pg_version|uint16|The PostgreSQL major version that send/recv format values will be compatible with. This is not necessarily the actual upstream PostgreSQL version.
+|binary.sizeof_int|uint8|sizeof(int) on the upstream.
+|binary.sizeof_long|uint8|sizeof(long) on the upstream.
+|binary.sizeof_datum|uint8|Same as sizeof_int, but for the PostgreSQL Datum typedef.
+|binary.maxalign|uint8|Upstream PostgreSQL server’s MAXIMUM_ALIGNOF value - platform dependent, determined at build time.
+|binary.bigendian|bool|True iff the upstream is big-endian.
+|binary.float4_byval|bool|Upstream PostgreSQL’s float4_byval compile option.
+|binary.float8_byval|bool|Upstream PostgreSQL’s float8_byval compile option.
+|binary.integer_datetimes|bool|Whether TIME, TIMESTAMP and TIMESTAMP WITH TIME ZONE will be sent using integer or floating point representation.
+
+Usually this is the value of the upstream PostgreSQL’s integer_datetimes compile option.
+|===
+== Startup errors
+
+If the server rejects the client’s connection - due to non-overlapping protocol
+support, unrecognised parameter formats, unsupported required parameters like
+hooks, etc - then it will follow the startup reply message with a
++++<u>+++normal libpq protocol error message+++</u>+++. (Current versions send
+this before the startup message).
+
+== Arguments client supplies to output plugin
+
+The one opportunity for the downstream client to send information (other than replay feedback) to the upstream is at connect-time, as an array of arguments to the output plugin supplied to START LOGICAL REPLICATION.
+
+There is no back-and-forth, no handshake.
+
+As a result, the client mainly announces capabilities and makes requests of the output plugin. The output plugin will ERROR if required parameters are unset, or where incompatibilities that cannot be resolved are found. Otherwise the output plugin reports what it could and could not honour in the startup message it sends as the first message on the wire down to the client. The client chooses whether to continue replay or to disconnect and report an error to the user, then possibly reconnect with different options.
+
+=== Output plugin arguments
+
+The output plugin’s key/value arguments are specified in pairs, as key and value. They’re what’s passed to START_REPLICATION, etc.
+
+All parameters are passed in text form. They _should_ be limited to 7-bit ASCII, since the server’s text encoding is not known, but _may_ be normalized precomposed UTF-8. The types specified for parameters indicate what the output plugin should attempt to convert the text into. Clients should not send text values that are outside the range for that type.
+
+==== Capabilities
+
+Many values are capabilities flags for the client, indicating that it understands optional features like metadata caching, binary format transfers, etc. In general the output plugin _may_ disregard capabilities the client advertises as supported and act as if they are not supported. If a capability is advertised as unsupported or is not advertised the output plugin _must not_ enable the corresponding features.
+
+In other words, don’t send the client something it’s not expecting.
+
+==== Protocol versioning
+
+Two parameters max_proto_version and min_proto_version, which clients must always send, allow negotiation of the protocol version. The output plugin must ERROR if the client protocol support does not overlap its own protocol support range.
+
+The protocol version is only incremented when there are major breaking changes that all or most clients must be modified to accommodate. Most changes are done by adding new optional messages and/or by having clients advertise capabilities to opt in to features.
+
+Because these versions are expected to be incremented, to make it clear that the format of the startup parameters themselves haven’t changed, the first key/value pair _must_ be the parameter startup_params_format with value “1”.
+
+|===
+|*Key*|*Type*|*Value(s)*|*Notes*
+
+|startup_params_format|int8|1|The format version of this startup parameter set. Always the digit 1 (0x31), null terminated.
+|max_proto_version|int32|1|Newest version of the protocol supported by client. Output plugin must ERROR if supported version too old. *Required*, ERROR if missing.
+|min_proto_version|int32|1|Oldest version of the protocol supported by client. Output plugin must ERROR if supported version too old. *Required*, ERROR if missing.
+|===
+
+==== Client requirements and capabilities
+
+|===
+|*Key*|*Type*|*Default*|*Notes*
+
+|expected_encoding|string|null|The text encoding the downstream expects field values to be in. Applies to text, binary and internal representations of field values in native format. Has no effect on other protocol content. If specified, the upstream must honour it. For json protocol, must be unset or match `client_encoding`. (Current plugin versions ERROR if this is set for the native protocol and not equal to the upstream database's encoding).
+|forward_changesets|bool|false|Request that all transactions, not just those originating on the upstream, be forwarded. See “_Changeset forwarding_”.
+|want_coltypes|boolean|false|The client wants to receive data type information about columns.
+|===
+
+==== General client information
+
+These keys tell the output plugin about the client. They’re mainly for informational purposes. In particular, the versions must _not_ be used to determine compatibility for binary or send/recv format, as non-PostgreSQL clients will simply not send them at all but may still understand binary or send/recv format fields.
+
+|===
+|*Key*|*Type*|*Default*|*Notes*
+
+|pg_version_num|integer|null|PostgreSQL server_version_num of client, if it’s PostgreSQL. e.g. 090400
+|pg_version|string|null|PostgreSQL server_version of client, if it’s PostgreSQL.
+|===
+
+
+==== Parameters relating to exchange of binary values
+
+The downstream may specify to the upstream that it is capable of understanding binary (PostgreSQL internal binary datum format), and/or send/recv (PostgreSQL binary interchange) format data by setting the binary.want_binary_basetypes and/or binary.want_internal_basetypes options, or other yet-to-be-defined options.
+
+An upstream output plugin that does not support one or both formats _may_ ignore the downstream’s binary support and send text format, in which case it may ignore all binary. parameters. All downstreams _must_ support text format. An upstream output plugin _must not_ send binary or send/recv format unless the downstream has announced it can receive it. If both upstream and downstream support both formats an upstream should prefer binary format and fall back to send/recv, then to text, if compatibility requires.
+
+Internal and binary format selection should be done on a type-by-type basis. It is quite normal to send ‘text’ format for extension types while sending binary for built-in types.
+
+The downstream _must_ specify its compatibility requirements for internal and binary data if it requests either or both formats. The upstream _must_ honour these by falling back from binary to send/recv, and from send/recv to text, where the upstream and downstream are not compatible.
+
+An unspecified compatibility field _must_ presumed to be unsupported by the downstream so that older clients that don’t know about a change in a newer version don’t receive unexpected data. For example, in the unlikely event that PostgreSQL 99.8 switched to 128-bit DPD (Densely Packed Decimal) representations of NUMERIC instead of the current arbitrary-length BCD (Binary Coded Decimal) format, a new binary.dpd_numerics parameter would be added. Clients that didn’t know about the change wouldn’t know to set it, so the upstream would presume it unsupported and send text format NUMERIC to those clients. This also means that clients that support the new format wouldn’t be able to receive the old format in binary from older servers since they’d specify dpd_numerics = true in their compatibility parameters.
+
+At this time a downstream may specify compatibility with only one value for a given option; i.e. a downstream cannot say it supports both 4-byte and 8-byte sizeof(int). Leaving it unspecified means the upstream must assume the downstream supports neither. (A future protocol extension may allow clients to specify alternative sets of supported formats).
+
+The `pg_version` option _must not_ be used to decide compatibility. Use `binary.basetypes_major_version` instead.
+
+|===
+|*Key name*|*Value type*|*Default*|*Description*
+
+|binary.want_binary_basetypes|boolean|false|True if the client accepts binary interchange (send/recv) format rows for PostgreSQL built-in base types.
+|binary.want_internal_basetypes|boolean|false|True if the client accepts PostgreSQL internal-format binary output for base PostgreSQL types not otherwise specified elsewhere.
+|binary.basetypes_major_version|uint16|null|The PostgreSQL major version (x.y) the downstream expects binary and send/recv format values to be in. Represented as an integer in XXYY format (no leading zero since it’s an integer), e.g. 9.5 is 905. This corresponds to PG_VERSION_NUM/100 in PostgreSQL.
+|binary.sizeof_int|uint8|+null+|sizeof(int) on the downstream.
+|binary.sizeof_long|uint8|null|sizeof(long) on the downstream.
+|binary.sizeof_datum|uint8|null|Same as sizeof_int, but for the PostgreSQL Datum typedef.
+|binary.maxalign|uint8|null|Downstream PostgreSQL server’s maxalign value - platform dependent, determined at build time.
+|binary.bigendian|bool|null|True iff the downstream is big-endian.
+|binary.float4_byval|bool|null|Downstream PostgreSQL’s float4_byval compile option.
+|binary.float8_byval|bool|null|Downstream PostgreSQL’s float8_byval compile option.
+|binary.integer_datetimes|bool|null|Downstream PostgreSQL’s integer_datetimes compile option.
+|===
+
+== Extensibility
+
+Because of the use of optional parameters in output plugin arguments, and the
+confirmation/response sent in the startup packet, a basic handshake is possible
+between upstream and downstream, allowing negotiation of capabilities.
+
+The output plugin must never send non-optional data or change its wire format
+without confirmation from the client that it can understand the new data. It
+may send optional data without negotiation.
+
+When extending the output plugin arguments, add-ons are expected to prefix all
+keys with the extension name, and should preferably use a single top level key
+with a json object value to carry their extension information. Additions to the
+startup message should follow the same pattern.
+
+Hooks and plugins can be used to add functionality specific to a client.
+
+== JSON protocol
+
+If `proto_format` is set to `json` then the output plugin will emit JSON
+instead of the custom binary protocol. JSON support is intended mainly for
+debugging and diagnostics.
+
+The JSON format supports all the same hooks.
diff --git a/contrib/pglogical_output/expected/basic_json.out b/contrib/pglogical_output/expected/basic_json.out
new file mode 100644
index 0000000..271189e
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_json.out
@@ -0,0 +1,139 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+--------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "f"
+ forward_changesets | "f"
+ hooks.row_filter_enabled | "f"
+ hooks.shutdown_hook_enabled | "f"
+ hooks.startup_hook_enabled | "f"
+ hooks.transaction_filter_enabled | "f"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+(19 rows)
+
+SELECT * FROM get_queued_data();
+ data
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "textval", "jsb": null, "seq": 1}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": "\\xdeadbeef0001", "js": null, "ts": null, "tx": null, "jsb": null, "seq": 2}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": "2045-09-12T12:34:56", "tx": "blah", "jsb": null, "seq": 3}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": {"key": "value"}, "ts": null, "tx": null, "jsb": {"key": "value"}, "seq": 4}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row1", "jsb": null, "seq": 6}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row2", "jsb": null, "seq": 7}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row3", "jsb": null, "seq": 8}, "relation": ["public", "demo"]}
+ {"action": "D", "oldtuple": {"ba": null, "js": null, "ts": null, "tx": null, "jsb": null, "seq": 7}, "relation": ["public", "demo"]}
+ {"action": "U", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "updated", "jsb": null, "seq": 6}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "I", "newtuple": {"id": 42}, "relation": ["public", "cat_test"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "1", "jsb": null, "seq": 9}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "2", "jsb": null, "seq": 10}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(28 rows)
+
+TRUNCATE TABLE json_decoding_output;
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/basic_json_1.out b/contrib/pglogical_output/expected/basic_json_1.out
new file mode 100644
index 0000000..293a8e6
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_json_1.out
@@ -0,0 +1,108 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+SELECT data::json
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ data
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action":"S", "params": {"max_proto_version":"1","min_proto_version":"1","coltypes":"f","pg_version_num":"90405","pg_version":"9.4.5","pg_catversion":"201409291","database_encoding":"UTF8","encoding":"UTF8","forward_changesets":"t","forward_changeset_origins":"f","binary.internal_basetypes":"f","binary.binary_basetypes":"f","binary.basetypes_major_version":"904","binary.sizeof_int":"4","binary.sizeof_long":"8","binary.sizeof_datum":"8","binary.maxalign":"8","binary.bigendian":"f","binary.float4_byval":"t","binary.float8_byval":"t","binary.integer_datetimes":"t","binary.binary_pg_version":"904","no_txinfo":"t","hooks.startup_hook_enabled":"f","hooks.shutdown_hook_enabled":"f","hooks.row_filter_enabled":"f","hooks.transaction_filter_enabled":"f"}}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":1,"tx":"textval","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":2,"tx":null,"ts":null,"jsb":null,"js":null,"ba":"\\xdeadbeef0001"}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":3,"tx":"blah","ts":"2045-09-12T12:34:56","jsb":null,"js":null,"ba":null}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":4,"tx":null,"ts":null,"jsb":{"key": "value"},"js":{"key":"value"},"ba":null}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":6,"tx":"row1","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":7,"tx":"row2","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":8,"tx":"row3","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"D","relation":["public","demo"],"oldtuple":{"seq":7,"tx":null,"ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"U","relation":["public","demo"],"newtuple":{"seq":6,"tx":"updated","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"t"}
+ {"action":"I","relation":["public","cat_test"],"newtuple":{"id":42}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":9,"tx":"1","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"I","relation":["public","demo"],"newtuple":{"seq":10,"tx":"2","ts":null,"jsb":null,"js":null,"ba":null}}
+ {"action":"C"}
+(27 rows)
+
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/basic_native.out b/contrib/pglogical_output/expected/basic_native.out
new file mode 100644
index 0000000..a7c88f3
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_native.out
@@ -0,0 +1,99 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ count
+-------
+ 39
+(1 row)
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+ count
+-------
+ 39
+(1 row)
+
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/cleanup.out b/contrib/pglogical_output/expected/cleanup.out
new file mode 100644
index 0000000..e7a02c8
--- /dev/null
+++ b/contrib/pglogical_output/expected/cleanup.out
@@ -0,0 +1,4 @@
+DROP TABLE excluded_startup_keys;
+DROP TABLE json_decoding_output;
+DROP FUNCTION get_queued_data();
+DROP FUNCTION get_startup_params();
diff --git a/contrib/pglogical_output/expected/encoding_json.out b/contrib/pglogical_output/expected/encoding_json.out
new file mode 100644
index 0000000..82c719a
--- /dev/null
+++ b/contrib/pglogical_output/expected/encoding_json.out
@@ -0,0 +1,59 @@
+SET synchronous_commit = on;
+-- This file doesn't share common setup with the native tests,
+-- since it's specific to how the text protocol handles encodings.
+CREATE TABLE enctest(blah text);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+SET client_encoding = 'UTF-8';
+INSERT INTO enctest(blah)
+VALUES
+('áàä'),('fl'), ('½⅓'), ('カンジ');
+RESET client_encoding;
+SET client_encoding = 'LATIN-1';
+-- Will ERROR, explicit encoding request doesn't match client_encoding
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: expected_encoding must be unset or match client_encoding in text protocols
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Will succeed since we don't request any encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: character with byte sequence 0xef 0xac 0x82 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+-- Will succeed since it matches the current encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'LATIN-1',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: character with byte sequence 0xef 0xac 0x82 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+RESET client_encoding;
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE enctest;
diff --git a/contrib/pglogical_output/expected/hooks_json.out b/contrib/pglogical_output/expected/hooks_json.out
new file mode 100644
index 0000000..f7a0839
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_json.out
@@ -0,0 +1,202 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Test table filter
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+--------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "f"
+ forward_changesets | "f"
+ hooks.row_filter_enabled | "t"
+ hooks.shutdown_hook_enabled | "t"
+ hooks.startup_hook_enabled | "t"
+ hooks.transaction_filter_enabled | "t"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+(19 rows)
+
+SELECT * FROM get_queued_data();
+ data
+---------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "U", "newtuple": {"id": 500}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(25 rows)
+
+-- test action filter
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output (ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+--------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "f"
+ forward_changesets | "f"
+ hooks.row_filter_enabled | "t"
+ hooks.shutdown_hook_enabled | "t"
+ hooks.startup_hook_enabled | "t"
+ hooks.transaction_filter_enabled | "t"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+(19 rows)
+
+SELECT * FROM get_queued_data();
+ data
+--------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_filter"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(36 rows)
+
+TRUNCATE TABLE json_decoding_output;
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/hooks_json_1.out b/contrib/pglogical_output/expected/hooks_json_1.out
new file mode 100644
index 0000000..4f4a0c7
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_json_1.out
@@ -0,0 +1,139 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Test table filter
+SELECT data::json
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ data
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action":"S", "params": {"max_proto_version":"1","min_proto_version":"1","coltypes":"f","pg_version_num":"90405","pg_version":"9.4.5","pg_catversion":"201409291","database_encoding":"UTF8","encoding":"UTF8","forward_changesets":"t","forward_changeset_origins":"f","binary.internal_basetypes":"f","binary.binary_basetypes":"f","binary.basetypes_major_version":"904","binary.sizeof_int":"4","binary.sizeof_long":"8","binary.sizeof_datum":"8","binary.maxalign":"8","binary.bigendian":"f","binary.float4_byval":"t","binary.float8_byval":"t","binary.integer_datetimes":"t","binary.binary_pg_version":"904","no_txinfo":"t","hooks.startup_hook_enabled":"t","hooks.shutdown_hook_enabled":"t","hooks.row_filter_enabled":"t","hooks.transaction_filter_enabled":"t"}}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":1}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":2}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":3}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":4}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":5}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":6}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":7}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":8}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":9}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":10}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"U","relation":["public","test_nofilt"],"newtuple":{"id":500}}
+ {"action":"C"}
+(24 rows)
+
+-- test action filter
+SELECT data::json
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ data
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action":"S", "params": {"max_proto_version":"1","min_proto_version":"1","coltypes":"f","pg_version_num":"90405","pg_version":"9.4.5","pg_catversion":"201409291","database_encoding":"UTF8","encoding":"UTF8","forward_changesets":"t","forward_changeset_origins":"f","binary.internal_basetypes":"f","binary.binary_basetypes":"f","binary.basetypes_major_version":"904","binary.sizeof_int":"4","binary.sizeof_long":"8","binary.sizeof_datum":"8","binary.maxalign":"8","binary.bigendian":"f","binary.float4_byval":"t","binary.float8_byval":"t","binary.integer_datetimes":"t","binary.binary_pg_version":"904","no_txinfo":"t","hooks.startup_hook_enabled":"t","hooks.shutdown_hook_enabled":"t","hooks.row_filter_enabled":"t","hooks.transaction_filter_enabled":"t"}}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":1}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":2}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":3}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":4}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":5}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":6}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":7}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":8}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":9}}
+ {"action":"I","relation":["public","test_filter"],"newtuple":{"id":10}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":1}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":2}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":3}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":4}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":5}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":6}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":7}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":8}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":9}}
+ {"action":"I","relation":["public","test_nofilt"],"newtuple":{"id":10}}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+ {"action":"B", "has_catalog_changes":"f"}
+ {"action":"C"}
+(33 rows)
+
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/hooks_native.out b/contrib/pglogical_output/expected/hooks_native.out
new file mode 100644
index 0000000..4a547cb
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_native.out
@@ -0,0 +1,104 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Regular hook setup
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+ count
+-------
+ 40
+(1 row)
+
+-- Test action filter
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+ count
+-------
+ 53
+(1 row)
+
+-- Invalid row fiter hook function
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+ERROR: function public.nosuchfunction(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Hook filter functoin with wrong signature
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+ERROR: function public.wrong_signature_fn(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/params_native.out b/contrib/pglogical_output/expected/params_native.out
new file mode 100644
index 0000000..9475035
--- /dev/null
+++ b/contrib/pglogical_output/expected/params_native.out
@@ -0,0 +1,118 @@
+SET synchronous_commit = on;
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+--
+-- Various invalid parameter combos:
+--
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: logical decoding output plugin "pglogical_output" produces binary output, but function "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+ERROR: client sent startup parameters in format 2 but we only support format 1
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: unrecognised encoding name bork passed to expected_encoding
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+ERROR: client requested protocol invalid but only "json" or "native" are supported
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+ data
+------
+(0 rows)
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
diff --git a/contrib/pglogical_output/expected/params_native_1.out b/contrib/pglogical_output/expected/params_native_1.out
new file mode 100644
index 0000000..e8d2745
--- /dev/null
+++ b/contrib/pglogical_output/expected/params_native_1.out
@@ -0,0 +1,118 @@
+SET synchronous_commit = on;
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+--
+-- Various invalid parameter combos:
+--
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: logical decoding output plugin "pglogical_output" produces binary output, but "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+ERROR: client sent startup parameters in format 2 but we only support format 1
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: unrecognised encoding name bork passed to expected_encoding
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+ERROR: client requested protocol invalid but only "json" or "native" are supported
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+ data
+------
+(0 rows)
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
diff --git a/contrib/pglogical_output/expected/prep.out b/contrib/pglogical_output/expected/prep.out
new file mode 100644
index 0000000..6501951
--- /dev/null
+++ b/contrib/pglogical_output/expected/prep.out
@@ -0,0 +1,26 @@
+CREATE TABLE excluded_startup_keys (key_name text primary key);
+INSERT INTO excluded_startup_keys
+VALUES
+('pg_version_num'),('pg_version'),('pg_catversion'),('binary.basetypes_major_version'),('binary.integer_datetimes'),('binary.bigendian'),('binary.maxalign'),('binary.binary_pg_version'),('sizeof_int'),('sizeof_long'),('sizeof_datum');
+CREATE UNLOGGED TABLE json_decoding_output(ch jsonb, rn integer);
+CREATE OR REPLACE FUNCTION get_startup_params()
+RETURNS TABLE ("key" text, "value" jsonb)
+LANGUAGE sql
+AS $$
+SELECT key, value
+FROM json_decoding_output
+CROSS JOIN LATERAL jsonb_each(ch -> 'params')
+WHERE rn = 1
+ AND key NOT IN (SELECT * FROM excluded_startup_keys)
+ AND ch ->> 'action' = 'S'
+ORDER BY key;
+$$;
+CREATE OR REPLACE FUNCTION get_queued_data()
+RETURNS TABLE (data jsonb)
+LANGUAGE sql
+AS $$
+SELECT ch
+FROM json_decoding_output
+WHERE rn > 1
+ORDER BY rn ASC;
+$$;
diff --git a/contrib/pglogical_output/pglogical_config.c b/contrib/pglogical_output/pglogical_config.c
new file mode 100644
index 0000000..cc22700
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.c
@@ -0,0 +1,499 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_config.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_config.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+
+#include "catalog/catversion.h"
+#include "catalog/namespace.h"
+
+#include "mb/pg_wchar.h"
+
+#include "nodes/makefuncs.h"
+
+#include "utils/builtins.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+typedef enum PGLogicalOutputParamType
+{
+ OUTPUT_PARAM_TYPE_BOOL,
+ OUTPUT_PARAM_TYPE_UINT32,
+ OUTPUT_PARAM_TYPE_STRING,
+ OUTPUT_PARAM_TYPE_QUALIFIED_NAME
+} PGLogicalOutputParamType;
+
+/* param parsing */
+static Datum get_param_value(DefElem *elem, bool null_ok,
+ PGLogicalOutputParamType type);
+
+static Datum get_param(List *options, const char *name, bool missing_ok,
+ bool null_ok, PGLogicalOutputParamType type,
+ bool *found);
+static bool parse_param_bool(DefElem *elem);
+static uint32 parse_param_uint32(DefElem *elem);
+
+static void
+process_parameters_v1(List *options, PGLogicalOutputData *data);
+
+enum {
+ PARAM_UNRECOGNISED,
+ PARAM_MAX_PROTOCOL_VERSION,
+ PARAM_MIN_PROTOCOL_VERSION,
+ PARAM_PROTOCOL_FORMAT,
+ PARAM_EXPECTED_ENCODING,
+ PARAM_BINARY_BIGENDIAN,
+ PARAM_BINARY_SIZEOF_DATUM,
+ PARAM_BINARY_SIZEOF_INT,
+ PARAM_BINARY_SIZEOF_LONG,
+ PARAM_BINARY_FLOAT4BYVAL,
+ PARAM_BINARY_FLOAT8BYVAL,
+ PARAM_BINARY_INTEGER_DATETIMES,
+ PARAM_BINARY_WANT_INTERNAL_BASETYPES,
+ PARAM_BINARY_WANT_BINARY_BASETYPES,
+ PARAM_BINARY_BASETYPES_MAJOR_VERSION,
+ PARAM_PG_VERSION,
+ PARAM_FORWARD_CHANGESETS,
+ PARAM_HOOKS_SETUP_FUNCTION,
+ PARAM_NO_TXINFO
+} OutputPluginParamKey;
+
+typedef struct {
+ const char * const paramname;
+ int paramkey;
+} OutputPluginParam;
+
+/* Oh, if only C had switch on strings */
+static OutputPluginParam param_lookup[] = {
+ {"max_proto_version", PARAM_MAX_PROTOCOL_VERSION},
+ {"min_proto_version", PARAM_MIN_PROTOCOL_VERSION},
+ {"proto_format", PARAM_PROTOCOL_FORMAT},
+ {"expected_encoding", PARAM_EXPECTED_ENCODING},
+ {"binary.bigendian", PARAM_BINARY_BIGENDIAN},
+ {"binary.sizeof_datum", PARAM_BINARY_SIZEOF_DATUM},
+ {"binary.sizeof_int", PARAM_BINARY_SIZEOF_INT},
+ {"binary.sizeof_long", PARAM_BINARY_SIZEOF_LONG},
+ {"binary.float4_byval", PARAM_BINARY_FLOAT4BYVAL},
+ {"binary.float8_byval", PARAM_BINARY_FLOAT8BYVAL},
+ {"binary.integer_datetimes", PARAM_BINARY_INTEGER_DATETIMES},
+ {"binary.want_internal_basetypes", PARAM_BINARY_WANT_INTERNAL_BASETYPES},
+ {"binary.want_binary_basetypes", PARAM_BINARY_WANT_BINARY_BASETYPES},
+ {"binary.basetypes_major_version", PARAM_BINARY_BASETYPES_MAJOR_VERSION},
+ {"pg_version", PARAM_PG_VERSION},
+ {"forward_changesets", PARAM_FORWARD_CHANGESETS},
+ {"hooks.setup_function", PARAM_HOOKS_SETUP_FUNCTION},
+ {"no_txinfo", PARAM_NO_TXINFO},
+ {NULL, PARAM_UNRECOGNISED}
+};
+
+/*
+ * Look up a param name to find the enum value for the
+ * param, or PARAM_UNRECOGNISED if not found.
+ */
+static int
+get_param_key(const char * const param_name)
+{
+ OutputPluginParam *param = ¶m_lookup[0];
+
+ do {
+ if (strcmp(param->paramname, param_name) == 0)
+ return param->paramkey;
+ param++;
+ } while (param->paramname != NULL);
+
+ return PARAM_UNRECOGNISED;
+}
+
+
+void
+process_parameters_v1(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ ListCell *lc;
+
+ /*
+ * max_proto_version and min_proto_version are specified
+ * as required, and must be parsed before anything else.
+ *
+ * TODO: We should still parse them as optional and
+ * delay the ERROR until after the startup reply.
+ */
+ val = get_param(options, "max_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_max_proto_version = DatumGetUInt32(val);
+
+ val = get_param(options, "min_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_min_proto_version = DatumGetUInt32(val);
+
+ /* Examine all the other params in the v1 message. */
+ foreach(lc, options)
+ {
+ DefElem *elem = lfirst(lc);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Check each param, whether or not we recognise it */
+ switch(get_param_key(elem->defname))
+ {
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+
+ case PARAM_BINARY_BIGENDIAN:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_bigendian_set = true;
+ data->client_binary_bigendian = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_DATUM:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofdatum = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_INT:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofint = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_LONG:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeoflong = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_FLOAT4BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_FLOAT8BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_INTEGER_DATETIMES:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_intdatetimes_set = true;
+ data->client_binary_intdatetimes = DatumGetBool(val);
+ break;
+
+ case PARAM_PROTOCOL_FORMAT:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_STRING);
+ data->client_protocol_format = DatumGetCString(val);
+ break;
+
+ case PARAM_EXPECTED_ENCODING:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_STRING);
+ data->client_expected_encoding = DatumGetCString(val);
+ break;
+
+ case PARAM_PG_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_pg_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_FORWARD_CHANGESETS:
+ /*
+ * Check to see if the client asked for changeset forwarding
+ *
+ * Note that we cannot support this on 9.4. We'll tell the client
+ * in the startup reply message.
+ */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_forward_changesets_set = true;
+ data->client_forward_changesets = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_WANT_INTERNAL_BASETYPES:
+ /* check if we want to use internal data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_internal_basetypes_set = true;
+ data->client_want_internal_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_WANT_BINARY_BASETYPES:
+ /* check if we want to use binary data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_binary_basetypes_set = true;
+ data->client_want_binary_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_BASETYPES_MAJOR_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_basetypes_major_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_HOOKS_SETUP_FUNCTION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_QUALIFIED_NAME);
+ data->hooks_setup_funcname = (List*) PointerGetDatum(val);
+ break;
+
+ case PARAM_NO_TXINFO:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_no_txinfo = DatumGetBool(val);
+ break;
+
+ case PARAM_UNRECOGNISED:
+ ereport(DEBUG1,
+ (errmsg("Unrecognised pglogical parameter %s ignored", elem->defname)));
+ break;
+ }
+ }
+}
+
+/*
+ * Read parameters sent by client at startup and store recognised
+ * ones in the parameters PGLogicalOutputData.
+ *
+ * The PGLogicalOutputData must have all client-surprised parameter fields
+ * zeroed, such as by memset or palloc0, since values not supplied
+ * by the client are not set.
+ */
+int
+process_parameters(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ int params_format;
+
+ val = get_param(options, "startup_params_format", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+
+ params_format = DatumGetUInt32(val);
+
+ if (params_format == 1)
+ {
+ process_parameters_v1(options, data);
+ }
+
+ return params_format;
+}
+
+static Datum
+get_param_value(DefElem *elem, bool null_ok, PGLogicalOutputParamType type)
+{
+ /* Check for NULL value */
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ {
+ if (null_ok)
+ return (Datum) 0;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("parameter \"%s\" cannot be NULL", elem->defname)));
+ }
+
+ switch (type)
+ {
+ case OUTPUT_PARAM_TYPE_UINT32:
+ return UInt32GetDatum(parse_param_uint32(elem));
+ case OUTPUT_PARAM_TYPE_BOOL:
+ return BoolGetDatum(parse_param_bool(elem));
+ case OUTPUT_PARAM_TYPE_STRING:
+ return PointerGetDatum(pstrdup(strVal(elem->arg)));
+ case OUTPUT_PARAM_TYPE_QUALIFIED_NAME:
+ return PointerGetDatum(textToQualifiedNameList(cstring_to_text(pstrdup(strVal(elem->arg)))));
+ default:
+ elog(ERROR, "unknown parameter type %d", type);
+ }
+}
+
+/*
+ * Param parsing
+ *
+ * This is not exactly fast but since it's only called on replication start
+ * we'll leave it for now.
+ */
+static Datum
+get_param(List *options, const char *name, bool missing_ok, bool null_ok,
+ PGLogicalOutputParamType type, bool *found)
+{
+ ListCell *option;
+
+ *found = false;
+
+ foreach(option, options)
+ {
+ DefElem *elem = lfirst(option);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Search until matching parameter found */
+ if (pg_strcasecmp(name, elem->defname))
+ continue;
+
+ *found = true;
+
+ return get_param_value(elem, null_ok, type);
+ }
+
+ if (!missing_ok)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("missing required parameter \"%s\"", name)));
+
+ return (Datum) 0;
+}
+
+static bool
+parse_param_bool(DefElem *elem)
+{
+ bool res;
+
+ if (!parse_bool(strVal(elem->arg), &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse boolean value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return res;
+}
+
+static uint32
+parse_param_uint32(DefElem *elem)
+{
+ int64 res;
+
+ if (!scanint8(strVal(elem->arg), true, &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse integer value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ if (res > PG_UINT32_MAX || res < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("value \"%s\" out of range for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return (uint32) res;
+}
+
+static List*
+add_startup_msg_s(List *l, char *key, char *val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(val)));
+}
+
+static List*
+add_startup_msg_i(List *l, char *key, int val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(psprintf("%d", val))));
+}
+
+static List*
+add_startup_msg_b(List *l, char *key, bool val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(val ? "t" : "f")));
+}
+
+/*
+ * This builds the protocol startup message, which is always the first
+ * message on the wire after the client sends START_REPLICATION.
+ *
+ * It confirms to the client that we could apply requested options, and
+ * tells the client our capabilities.
+ *
+ * Any additional parameters provided by the startup hook are also output
+ * now.
+ *
+ * The output param 'msg' is a null-terminated char* palloc'd in the current
+ * memory context and the length 'len' of that string that is valid. The caller
+ * should pfree the result after use.
+ *
+ * This is a bit less efficient than direct pq_sendblah calls, but
+ * separates config handling from the protocol implementation, and
+ * it's not like startup msg performance matters much.
+ */
+List *
+prepare_startup_message(PGLogicalOutputData *data)
+{
+ ListCell *lc;
+ List *l = NIL;
+
+ l = add_startup_msg_s(l, "max_proto_version", "1");
+ l = add_startup_msg_s(l, "min_proto_version", "1");
+
+ /* We don't support understand column types yet */
+ l = add_startup_msg_b(l, "coltypes", false);
+
+ /* Info about our Pg host */
+ l = add_startup_msg_i(l, "pg_version_num", PG_VERSION_NUM);
+ l = add_startup_msg_s(l, "pg_version", PG_VERSION);
+ l = add_startup_msg_i(l, "pg_catversion", CATALOG_VERSION_NO);
+
+ l = add_startup_msg_s(l, "database_encoding", (char*)GetDatabaseEncodingName());
+
+ l = add_startup_msg_s(l, "encoding", (char*)pg_encoding_to_char(data->field_datum_encoding));
+
+ l = add_startup_msg_b(l, "forward_changesets",
+ data->forward_changesets);
+ l = add_startup_msg_b(l, "forward_changeset_origins",
+ data->forward_changeset_origins);
+
+ /* binary options enabled */
+ l = add_startup_msg_b(l, "binary.internal_basetypes",
+ data->allow_internal_basetypes);
+ l = add_startup_msg_b(l, "binary.binary_basetypes",
+ data->allow_binary_basetypes);
+
+ /* Binary format characteristics of server */
+ l = add_startup_msg_i(l, "binary.basetypes_major_version", PG_VERSION_NUM/100);
+ l = add_startup_msg_i(l, "binary.sizeof_int", sizeof(int));
+ l = add_startup_msg_i(l, "binary.sizeof_long", sizeof(long));
+ l = add_startup_msg_i(l, "binary.sizeof_datum", sizeof(Datum));
+ l = add_startup_msg_i(l, "binary.maxalign", MAXIMUM_ALIGNOF);
+ l = add_startup_msg_b(l, "binary.bigendian", server_bigendian());
+ l = add_startup_msg_b(l, "binary.float4_byval", server_float4_byval());
+ l = add_startup_msg_b(l, "binary.float8_byval", server_float8_byval());
+ l = add_startup_msg_b(l, "binary.integer_datetimes", server_integer_datetimes());
+ /* We don't know how to send in anything except our host's format */
+ l = add_startup_msg_i(l, "binary.binary_pg_version",
+ PG_VERSION_NUM/100);
+
+ l = add_startup_msg_b(l, "no_txinfo", data->client_no_txinfo);
+
+
+ /*
+ * Confirm that we've enabled any requested hook functions.
+ */
+ l = add_startup_msg_b(l, "hooks.startup_hook_enabled",
+ data->hooks.startup_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.shutdown_hook_enabled",
+ data->hooks.shutdown_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.row_filter_enabled",
+ data->hooks.row_filter_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.transaction_filter_enabled",
+ data->hooks.txn_filter_hook != NULL);
+
+ /*
+ * Output any extra params supplied by a startup hook by appending
+ * them verbatim to the params list.
+ */
+ foreach(lc, data->extra_startup_params)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ l = lappend(l, param);
+ }
+
+ return l;
+}
diff --git a/contrib/pglogical_output/pglogical_config.h b/contrib/pglogical_output/pglogical_config.h
new file mode 100644
index 0000000..3af3ce8
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.h
@@ -0,0 +1,55 @@
+#ifndef PG_LOGICAL_CONFIG_H
+#define PG_LOGICAL_CONFIG_H
+
+#ifndef PG_VERSION_NUM
+#error <postgres.h> must be included first
+#endif
+
+inline static bool
+server_float4_byval(void)
+{
+#ifdef USE_FLOAT4_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_float8_byval(void)
+{
+#ifdef USE_FLOAT8_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_integer_datetimes(void)
+{
+#ifdef USE_INTEGER_DATETIMES
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_bigendian(void)
+{
+#ifdef WORDS_BIGENDIAN
+ return true;
+#else
+ return false;
+#endif
+}
+
+typedef struct List List;
+typedef struct PGLogicalOutputData PGLogicalOutputData;
+
+extern int process_parameters(List *options, PGLogicalOutputData *data);
+
+extern List * prepare_startup_message(PGLogicalOutputData *data);
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_hooks.c b/contrib/pglogical_output/pglogical_hooks.c
new file mode 100644
index 0000000..73e8120
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.c
@@ -0,0 +1,232 @@
+#include "postgres.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#include "replication/origin.h"
+
+#include "parser/parse_func.h"
+
+#include "utils/acl.h"
+#include "utils/lsyscache.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_hooks.h"
+#include "pglogical_output.h"
+
+/*
+ * Returns Oid of the hooks function specified in funcname.
+ *
+ * Error is thrown if function doesn't exist or doen't return correct datatype
+ * or is volatile.
+ */
+static Oid
+get_hooks_function_oid(List *funcname)
+{
+ Oid funcid;
+ Oid funcargtypes[1];
+
+ funcargtypes[0] = INTERNALOID;
+
+ /* find the the function */
+ funcid = LookupFuncName(funcname, 1, funcargtypes, false);
+
+ /* Validate that the function returns void */
+ if (get_func_rettype(funcid) != VOIDOID)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must return void",
+ NameListToString(funcname))));
+ }
+
+ if (func_volatile(funcid) == PROVOLATILE_VOLATILE)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must not be VOLATILE",
+ NameListToString(funcname))));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(funcname))));
+ }
+
+ return funcid;
+}
+
+/*
+ * If a hook setup function was specified in the startup parameters, look it up
+ * in the catalogs, check permissions, call it, and store the resulting hook
+ * info struct.
+ */
+void
+load_hooks(PGLogicalOutputData *data)
+{
+ Oid hooks_func;
+ MemoryContext old_ctxt;
+ bool txn_started = false;
+
+ if (!IsTransactionState())
+ {
+ txn_started = true;
+ StartTransactionCommand();
+ }
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+ hooks_func = get_hooks_function_oid(data->hooks_setup_funcname);
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) OidFunctionCall1(hooks_func, PointerGetDatum(&data->hooks));
+ MemoryContextSwitchTo(old_ctxt);
+
+ elog(DEBUG3, "pglogical_output: Loaded hooks from function %u. Hooks are: \n"
+ "\tstartup_hook: %p\n"
+ "\tshutdown_hook: %p\n"
+ "\trow_filter_hook: %p\n"
+ "\ttxn_filter_hook: %p\n"
+ "\thooks_private_data: %p\n",
+ hooks_func,
+ data->hooks.startup_hook,
+ data->hooks.shutdown_hook,
+ data->hooks.row_filter_hook,
+ data->hooks.txn_filter_hook,
+ data->hooks.hooks_private_data);
+ }
+
+ if (txn_started)
+ CommitTransactionCommand();
+}
+
+void
+call_startup_hook(PGLogicalOutputData *data, List *plugin_params)
+{
+ struct PGLogicalStartupHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.startup_hook != NULL)
+ {
+ bool tx_started = false;
+
+ args.private_data = data->hooks.hooks_private_data;
+ args.in_params = plugin_params;
+ args.out_params = NIL;
+
+ elog(DEBUG3, "calling pglogical startup hook");
+
+ if (!IsTransactionState())
+ {
+ tx_started = true;
+ StartTransactionCommand();
+ }
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.startup_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ if (tx_started)
+ CommitTransactionCommand();
+
+ data->extra_startup_params = args.out_params;
+ /* The startup hook might change the private data seg */
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical startup hook");
+ }
+}
+
+void
+call_shutdown_hook(PGLogicalOutputData *data)
+{
+ struct PGLogicalShutdownHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.shutdown_hook != NULL)
+ {
+ args.private_data = data->hooks.hooks_private_data;
+
+ elog(DEBUG3, "calling pglogical shutdown hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.shutdown_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical shutdown hook");
+ }
+}
+
+/*
+ * Decide if the individual change should be filtered out by
+ * calling a client-provided hook.
+ */
+bool
+call_row_filter_hook(PGLogicalOutputData *data, ReorderBufferTXN *txn,
+ Relation rel, ReorderBufferChange *change)
+{
+ struct PGLogicalRowFilterArgs hook_args;
+ MemoryContext old_ctxt;
+ bool ret = true;
+
+ if (data->hooks.row_filter_hook != NULL)
+ {
+ hook_args.change_type = change->action;
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.changed_rel = rel;
+
+ elog(DEBUG3, "calling pglogical row filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.row_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical row filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
+
+bool
+call_txn_filter_hook(PGLogicalOutputData *data, RepOriginId txn_origin)
+{
+ struct PGLogicalTxnFilterArgs hook_args;
+ bool ret = true;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.txn_filter_hook != NULL)
+ {
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.origin_id = txn_origin;
+
+ elog(DEBUG3, "calling pglogical txn filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.txn_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical txn filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
diff --git a/contrib/pglogical_output/pglogical_hooks.h b/contrib/pglogical_output/pglogical_hooks.h
new file mode 100644
index 0000000..df661f3
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.h
@@ -0,0 +1,22 @@
+#ifndef PGLOGICAL_HOOKS_H
+#define PGLOGICAL_HOOKS_H
+
+#include "replication/reorderbuffer.h"
+
+/* public interface for hooks */
+#include "pglogical_output/hooks.h"
+
+extern void load_hooks(PGLogicalOutputData *data);
+
+extern void call_startup_hook(PGLogicalOutputData *data, List *plugin_params);
+
+extern void call_shutdown_hook(PGLogicalOutputData *data);
+
+extern bool call_row_filter_hook(PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, Relation rel, ReorderBufferChange *change);
+
+extern bool call_txn_filter_hook(PGLogicalOutputData *data,
+ RepOriginId txn_origin);
+
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_output.c b/contrib/pglogical_output/pglogical_output.c
new file mode 100644
index 0000000..b8fc55e
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.c
@@ -0,0 +1,537 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+#include "pglogical_hooks.h"
+
+#include "access/hash.h"
+#include "access/sysattr.h"
+#include "access/xact.h"
+
+#include "catalog/pg_class.h"
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#include "mb/pg_wchar.h"
+
+#include "nodes/parsenodes.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/output_plugin.h"
+#include "replication/logical.h"
+#include "replication/origin.h"
+
+#include "utils/builtins.h"
+#include "utils/catcache.h"
+#include "utils/guc.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+PG_MODULE_MAGIC;
+
+extern void _PG_output_plugin_init(OutputPluginCallbacks *cb);
+
+/* These must be available to pg_dlsym() */
+static void pg_decode_startup(LogicalDecodingContext * ctx,
+ OutputPluginOptions *opt, bool is_init);
+static void pg_decode_shutdown(LogicalDecodingContext * ctx);
+static void pg_decode_begin_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn);
+static void pg_decode_commit_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+static void pg_decode_change(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, Relation rel,
+ ReorderBufferChange *change);
+
+static bool pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id);
+
+static void send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message);
+
+static bool startup_message_sent = false;
+
+/* specify output plugin callbacks */
+void
+_PG_output_plugin_init(OutputPluginCallbacks *cb)
+{
+ AssertVariableIsOfType(&_PG_output_plugin_init, LogicalOutputPluginInit);
+
+ cb->startup_cb = pg_decode_startup;
+ cb->begin_cb = pg_decode_begin_txn;
+ cb->change_cb = pg_decode_change;
+ cb->commit_cb = pg_decode_commit_txn;
+ cb->filter_by_origin_cb = pg_decode_origin_filter;
+ cb->shutdown_cb = pg_decode_shutdown;
+}
+
+static bool
+check_binary_compatibility(PGLogicalOutputData *data)
+{
+ if (data->client_binary_basetypes_major_version != PG_VERSION_NUM / 100)
+ return false;
+
+ if (data->client_binary_bigendian_set
+ && data->client_binary_bigendian != server_bigendian())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian mis-match");
+ return false;
+ }
+
+ if (data->client_binary_sizeofdatum != 0
+ && data->client_binary_sizeofdatum != sizeof(Datum))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(Datum) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeofint != 0
+ && data->client_binary_sizeofint != sizeof(int))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(int) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeoflong != 0
+ && data->client_binary_sizeoflong != sizeof(long))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(long) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float4byval_set
+ && data->client_binary_float4byval != server_float4_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float4byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float8byval_set
+ && data->client_binary_float8byval != server_float8_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float8byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_intdatetimes_set
+ && data->client_binary_intdatetimes != server_integer_datetimes())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian integer datetimes mismatch");
+ return false;
+ }
+
+ return true;
+}
+
+/* initialize this plugin */
+static void
+pg_decode_startup(LogicalDecodingContext * ctx, OutputPluginOptions *opt,
+ bool is_init)
+{
+ PGLogicalOutputData *data = palloc0(sizeof(PGLogicalOutputData));
+
+ data->context = AllocSetContextCreate(TopMemoryContext,
+ "pglogical conversion context",
+ ALLOCSET_DEFAULT_MINSIZE,
+ ALLOCSET_DEFAULT_INITSIZE,
+ ALLOCSET_DEFAULT_MAXSIZE);
+ data->allow_internal_basetypes = false;
+ data->allow_binary_basetypes = false;
+
+
+ ctx->output_plugin_private = data;
+
+ /*
+ * This is replication start and not slot initialization.
+ *
+ * Parse and validate options passed by the client.
+ */
+ if (!is_init)
+ {
+ int params_format;
+
+ /*
+ * Ideally we'd send the startup message immediately. That way
+ * it'd arrive before any error we emit if we see incompatible
+ * options sent by the client here. That way the client could
+ * possibly adjust its options and reconnect. It'd also make
+ * sure the client gets the startup message in a timely way if
+ * the server is idle, since otherwise it could be a while
+ * before the next callback.
+ *
+ * The decoding plugin API doesn't let us write to the stream
+ * from here, though, so we have to delay the startup message
+ * until the first change processed on the stream, in a begin
+ * callback.
+ *
+ * If we ERROR there, the startup message is buffered but not
+ * sent since the callback didn't finish. So we'd have to send
+ * the startup message, finish the callback and check in the
+ * next callback if we need to ERROR.
+ *
+ * That's a bit much hoop jumping, so for now ERRORs are
+ * immediate. A way to emit a message from the startup callback
+ * is really needed to change that.
+ */
+ startup_message_sent = false;
+
+ /* Now parse the rest of the params and ERROR if we see any we don't recognise */
+ params_format = process_parameters(ctx->output_plugin_options, data);
+
+ if (params_format != 1)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent startup parameters in format %d but we only support format 1",
+ params_format)));
+
+ if (data->client_min_proto_version > PG_LOGICAL_PROTO_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent min_proto_version=%d but we only support protocol %d or lower",
+ data->client_min_proto_version, PG_LOGICAL_PROTO_VERSION_NUM)));
+
+ if (data->client_max_proto_version < PG_LOGICAL_PROTO_MIN_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent max_proto_version=%d but we only support protocol %d or higher",
+ data->client_max_proto_version, PG_LOGICAL_PROTO_MIN_VERSION_NUM)));
+
+ /*
+ * Set correct protocol format.
+ *
+ * This is the output plugin protocol format, this is different
+ * from the individual fields binary vs textual format.
+ */
+ if (data->client_protocol_format != NULL
+ && strcmp(data->client_protocol_format, "json") == 0)
+ {
+ data->api = pglogical_init_api(PGLogicalProtoJson);
+ opt->output_type = OUTPUT_PLUGIN_TEXTUAL_OUTPUT;
+ }
+ else if ((data->client_protocol_format != NULL
+ && strcmp(data->client_protocol_format, "native") == 0)
+ || data->client_protocol_format == NULL)
+ {
+ data->api = pglogical_init_api(PGLogicalProtoNative);
+ opt->output_type = OUTPUT_PLUGIN_BINARY_OUTPUT;
+
+ if (data->client_no_txinfo)
+ {
+ elog(WARNING, "no_txinfo option ignored for protocols other than json");
+ data->client_no_txinfo = false;
+ }
+ }
+ else
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client requested protocol %s but only \"json\" or \"native\" are supported",
+ data->client_protocol_format)));
+ }
+
+ /* check for encoding match if specific encoding demanded by client */
+ if (data->client_expected_encoding != NULL
+ && strlen(data->client_expected_encoding) != 0)
+ {
+ int wanted_encoding = pg_char_to_encoding(data->client_expected_encoding);
+
+ if (wanted_encoding == -1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("unrecognised encoding name %s passed to expected_encoding",
+ data->client_expected_encoding)));
+
+ if (opt->output_type == OUTPUT_PLUGIN_TEXTUAL_OUTPUT)
+ {
+ /*
+ * datum encoding must match assigned client_encoding in text
+ * proto, since everything is subject to client_encoding
+ * conversion.
+ */
+ if (wanted_encoding != pg_get_client_encoding())
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("expected_encoding must be unset or match client_encoding in text protocols")));
+ }
+ else
+ {
+ /*
+ * currently in the binary protocol we can only emit encoded
+ * datums in the server encoding. There's no support for encoding
+ * conversion.
+ */
+ if (wanted_encoding != GetDatabaseEncoding())
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("encoding conversion for binary datum not supported yet"),
+ errdetail("expected_encoding %s must be unset or match server_encoding %s",
+ data->client_expected_encoding, GetDatabaseEncodingName())));
+ }
+
+ data->field_datum_encoding = wanted_encoding;
+ }
+
+ /*
+ * It's obviously not possible to send binary representatio of data
+ * unless we use the binary output.
+ */
+ if (opt->output_type == OUTPUT_PLUGIN_BINARY_OUTPUT &&
+ data->client_want_internal_basetypes)
+ {
+ data->allow_internal_basetypes =
+ check_binary_compatibility(data);
+ }
+
+ if (opt->output_type == OUTPUT_PLUGIN_BINARY_OUTPUT &&
+ data->client_want_binary_basetypes &&
+ data->client_binary_basetypes_major_version == PG_VERSION_NUM / 100)
+ {
+ data->allow_binary_basetypes = true;
+ }
+
+ /*
+ * Will we forward changesets? We have to if we're on 9.4;
+ * otherwise honour the client's request.
+ */
+ if (PG_VERSION_NUM/100 == 904)
+ {
+ /*
+ * 9.4 unconditionally forwards changesets due to lack of
+ * replication origins, and it can't ever send origin info
+ * for the same reason.
+ */
+ data->forward_changesets = true;
+ data->forward_changeset_origins = false;
+
+ if (data->client_forward_changesets_set
+ && !data->client_forward_changesets)
+ {
+ ereport(DEBUG1,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("Cannot disable changeset forwarding on PostgreSQL 9.4")));
+ }
+ }
+ else if (data->client_forward_changesets_set
+ && data->client_forward_changesets)
+ {
+ /* Client explicitly asked for forwarding; forward csets and origins */
+ data->forward_changesets = true;
+ data->forward_changeset_origins = true;
+ }
+ else
+ {
+ /* Default to not forwarding or honour client's request not to fwd */
+ data->forward_changesets = false;
+ data->forward_changeset_origins = false;
+ }
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+
+ data->hooks_mctxt = AllocSetContextCreate(ctx->context,
+ "pglogical_output hooks context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+ load_hooks(data);
+ call_startup_hook(data, ctx->output_plugin_options);
+ }
+ }
+}
+
+/*
+ * BEGIN callback
+ */
+void
+pg_decode_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+ bool send_replication_origin = data->forward_changeset_origins;
+
+ if (!startup_message_sent)
+ send_startup_message(ctx, data, false /* can't be last message */);
+
+ /* If the record didn't originate locally, send origin info */
+ send_replication_origin &= txn->origin_id != InvalidRepOriginId;
+
+ OutputPluginPrepareWrite(ctx, !send_replication_origin);
+ data->api->write_begin(ctx->out, data, txn);
+
+ if (send_replication_origin)
+ {
+ char *origin;
+
+ /* Message boundary */
+ OutputPluginWrite(ctx, false);
+ OutputPluginPrepareWrite(ctx, true);
+
+ /*
+ * XXX: which behaviour we want here?
+ *
+ * Alternatives:
+ * - don't send origin message if origin name not found
+ * (that's what we do now)
+ * - throw error - that will break replication, not good
+ * - send some special "unknown" origin
+ */
+ if (data->api->write_origin &&
+ replorigin_by_oid(txn->origin_id, true, &origin))
+ data->api->write_origin(ctx->out, origin, txn->origin_lsn);
+ }
+
+ OutputPluginWrite(ctx, true);
+}
+
+/*
+ * COMMIT callback
+ */
+void
+pg_decode_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_commit(ctx->out, data, txn, commit_lsn);
+ OutputPluginWrite(ctx, true);
+}
+
+void
+pg_decode_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ Relation relation, ReorderBufferChange *change)
+{
+ PGLogicalOutputData *data = ctx->output_plugin_private;
+ MemoryContext old;
+
+ /* First check the table filter */
+ if (!call_row_filter_hook(data, txn, relation, change))
+ return;
+
+ /* Avoid leaking memory by using and resetting our own context */
+ old = MemoryContextSwitchTo(data->context);
+
+ /* TODO: add caching (send only if changed) */
+ if (data->api->write_rel)
+ {
+ OutputPluginPrepareWrite(ctx, false);
+ data->api->write_rel(ctx->out, relation);
+ OutputPluginWrite(ctx, false);
+ }
+
+ /* Send the data */
+ switch (change->action)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_insert(ctx->out, data, relation,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ {
+ HeapTuple oldtuple = change->data.tp.oldtuple ?
+ &change->data.tp.oldtuple->tuple : NULL;
+
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_update(ctx->out, data, relation, oldtuple,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ }
+ case REORDER_BUFFER_CHANGE_DELETE:
+ if (change->data.tp.oldtuple)
+ {
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_delete(ctx->out, data, relation,
+ &change->data.tp.oldtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ }
+ else
+ elog(DEBUG1, "didn't send DELETE change because of missing oldtuple");
+ break;
+ default:
+ Assert(false);
+ }
+
+ /* Cleanup */
+ MemoryContextSwitchTo(old);
+ MemoryContextReset(data->context);
+}
+
+/*
+ * Decide if the whole transaction with specific origin should be filtered out.
+ */
+static bool
+pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id)
+{
+ PGLogicalOutputData *data = ctx->output_plugin_private;
+
+ if (!call_txn_filter_hook(data, origin_id))
+ return true;
+
+ if (!data->forward_changesets && origin_id != InvalidRepOriginId)
+ return true;
+
+ return false;
+}
+
+static void
+send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message)
+{
+ List *msg;
+
+ Assert(!startup_message_sent);
+
+ msg = prepare_startup_message(data);
+
+ /*
+ * We could free the extra_startup_params DefElem list here, but it's
+ * pretty harmless to just ignore it, since it's in the decoding memory
+ * context anyway, and we don't know if it's safe to free the defnames or
+ * not.
+ */
+
+ OutputPluginPrepareWrite(ctx, last_message);
+ data->api->write_startup_message(ctx->out, msg);
+ OutputPluginWrite(ctx, last_message);
+
+ pfree(msg);
+
+ startup_message_sent = true;
+}
+
+static void pg_decode_shutdown(LogicalDecodingContext * ctx)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+
+ call_shutdown_hook(data);
+
+ if (data->hooks_mctxt != NULL)
+ {
+ MemoryContextDelete(data->hooks_mctxt);
+ data->hooks_mctxt = NULL;
+ }
+}
diff --git a/contrib/pglogical_output/pglogical_output.h b/contrib/pglogical_output/pglogical_output.h
new file mode 100644
index 0000000..a874c40
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.h
@@ -0,0 +1,105 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.h
+ * pglogical output plugin
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_OUTPUT_H
+#define PG_LOGICAL_OUTPUT_H
+
+#include "nodes/parsenodes.h"
+
+#include "replication/logical.h"
+#include "replication/output_plugin.h"
+
+#include "storage/lock.h"
+
+#include "pglogical_output/hooks.h"
+
+#include "pglogical_proto.h"
+
+#define PG_LOGICAL_PROTO_VERSION_NUM 1
+#define PG_LOGICAL_PROTO_MIN_VERSION_NUM 1
+
+/*
+ * The name of a hook function. This is used instead of the usual List*
+ * because can serve as a hash key.
+ *
+ * Must be zeroed on allocation if used as a hash key since padding is
+ * *not* ignored on compare.
+ */
+typedef struct HookFuncName
+{
+ /* funcname is more likely to be unique, so goes first */
+ char function[NAMEDATALEN];
+ char schema[NAMEDATALEN];
+} HookFuncName;
+
+typedef struct PGLogicalOutputData
+{
+ MemoryContext context;
+
+ PGLogicalProtoAPI *api;
+
+ /* protocol */
+ bool allow_internal_basetypes;
+ bool allow_binary_basetypes;
+ bool forward_changesets;
+ bool forward_changeset_origins;
+ int field_datum_encoding;
+
+ /*
+ * client info
+ *
+ * Lots of this should move to a separate shorter-lived struct used only
+ * during parameter reading, since it contains what the client asked for.
+ * Once we've processed this during startup we don't refer to it again.
+ */
+ uint32 client_pg_version;
+ uint32 client_max_proto_version;
+ uint32 client_min_proto_version;
+ const char *client_expected_encoding;
+ const char *client_protocol_format;
+ uint32 client_binary_basetypes_major_version;
+ bool client_want_internal_basetypes_set;
+ bool client_want_internal_basetypes;
+ bool client_want_binary_basetypes_set;
+ bool client_want_binary_basetypes;
+ bool client_binary_bigendian_set;
+ bool client_binary_bigendian;
+ uint32 client_binary_sizeofdatum;
+ uint32 client_binary_sizeofint;
+ uint32 client_binary_sizeoflong;
+ bool client_binary_float4byval_set;
+ bool client_binary_float4byval;
+ bool client_binary_float8byval_set;
+ bool client_binary_float8byval;
+ bool client_binary_intdatetimes_set;
+ bool client_binary_intdatetimes;
+ bool client_forward_changesets_set;
+ bool client_forward_changesets;
+ bool client_no_txinfo;
+
+ /* hooks */
+ List *hooks_setup_funcname;
+ struct PGLogicalHooks hooks;
+ MemoryContext hooks_mctxt;
+
+ /* DefElem<String> list populated by startup hook */
+ List *extra_startup_params;
+} PGLogicalOutputData;
+
+typedef struct PGLogicalTupleData
+{
+ Datum values[MaxTupleAttributeNumber];
+ bool nulls[MaxTupleAttributeNumber];
+ bool changed[MaxTupleAttributeNumber];
+} PGLogicalTupleData;
+
+#endif /* PG_LOGICAL_OUTPUT_H */
diff --git a/contrib/pglogical_output/pglogical_output/README b/contrib/pglogical_output/pglogical_output/README
new file mode 100644
index 0000000..5480e5c
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/README
@@ -0,0 +1,7 @@
+/*
+ * This directory contains the public header files for the pglogical_output
+ * extension. It is installed into the PostgreSQL source tree when the extension
+ * is installed.
+ *
+ * These headers are not part of the PostgreSQL project its self.
+ */
diff --git a/contrib/pglogical_output/pglogical_output/hooks.h b/contrib/pglogical_output/pglogical_output/hooks.h
new file mode 100644
index 0000000..b20fa72
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/hooks.h
@@ -0,0 +1,72 @@
+#ifndef PGLOGICAL_OUTPUT_HOOKS_H
+#define PGLOGICAL_OUTPUT_HOOKS_H
+
+#include "access/xlogdefs.h"
+#include "nodes/pg_list.h"
+#include "utils/rel.h"
+#include "utils/palloc.h"
+#include "replication/reorderbuffer.h"
+
+struct PGLogicalOutputData;
+typedef struct PGLogicalOutputData PGLogicalOutputData;
+
+/*
+ * This header is to be included by extensions that implement pglogical output
+ * plugin callback hooks for transaction origin and row filtering, etc. It is
+ * installed as "pglogical_output/hooks.h"
+ *
+ * See the README.md and the example in examples/hooks/ for details on hooks.
+ */
+
+
+struct PGLogicalStartupHookArgs
+{
+ void *private_data;
+ List *in_params;
+ List *out_params;
+};
+
+typedef void (*pglogical_startup_hook_fn)(struct PGLogicalStartupHookArgs *args);
+
+
+struct PGLogicalTxnFilterArgs
+{
+ void *private_data;
+ RepOriginId origin_id;
+};
+
+typedef bool (*pglogical_txn_filter_hook_fn)(struct PGLogicalTxnFilterArgs *args);
+
+
+struct PGLogicalRowFilterArgs
+{
+ void *private_data;
+ Relation changed_rel;
+ enum ReorderBufferChangeType change_type;
+};
+
+typedef bool (*pglogical_row_filter_hook_fn)(struct PGLogicalRowFilterArgs *args);
+
+
+struct PGLogicalShutdownHookArgs
+{
+ void *private_data;
+};
+
+typedef void (*pglogical_shutdown_hook_fn)(struct PGLogicalShutdownHookArgs *args);
+
+/*
+ * This struct is passed to the pglogical_get_hooks_fn as the first argument,
+ * typed 'internal', and is unwrapped with `DatumGetPointer`.
+ */
+struct PGLogicalHooks
+{
+ pglogical_startup_hook_fn startup_hook;
+ pglogical_shutdown_hook_fn shutdown_hook;
+ pglogical_txn_filter_hook_fn txn_filter_hook;
+ pglogical_row_filter_hook_fn row_filter_hook;
+ void *hooks_private_data;
+};
+
+
+#endif /* PGLOGICAL_OUTPUT_HOOKS_H */
diff --git a/contrib/pglogical_output/pglogical_proto.c b/contrib/pglogical_output/pglogical_proto.c
new file mode 100644
index 0000000..47a883f
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.c
@@ -0,0 +1,49 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.c
+ * pglogical protocol functions
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+#include "pglogical_proto_native.h"
+#include "pglogical_proto_json.h"
+
+PGLogicalProtoAPI *
+pglogical_init_api(PGLogicalProtoType typ)
+{
+ PGLogicalProtoAPI *res = palloc0(sizeof(PGLogicalProtoAPI));
+
+ if (typ == PGLogicalProtoJson)
+ {
+ res->write_rel = NULL;
+ res->write_begin = pglogical_json_write_begin;
+ res->write_commit = pglogical_json_write_commit;
+ res->write_origin = NULL;
+ res->write_insert = pglogical_json_write_insert;
+ res->write_update = pglogical_json_write_update;
+ res->write_delete = pglogical_json_write_delete;
+ res->write_startup_message = json_write_startup_message;
+ }
+ else
+ {
+ res->write_rel = pglogical_write_rel;
+ res->write_begin = pglogical_write_begin;
+ res->write_commit = pglogical_write_commit;
+ res->write_origin = pglogical_write_origin;
+ res->write_insert = pglogical_write_insert;
+ res->write_update = pglogical_write_update;
+ res->write_delete = pglogical_write_delete;
+ res->write_startup_message = write_startup_message;
+ }
+
+ return res;
+}
diff --git a/contrib/pglogical_output/pglogical_proto.h b/contrib/pglogical_output/pglogical_proto.h
new file mode 100644
index 0000000..b56ff6f
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.h
@@ -0,0 +1,57 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.h
+ * pglogical protocol
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_H
+#define PG_LOGICAL_PROTO_H
+
+typedef void (*pglogical_write_rel_fn)(StringInfo out, Relation rel);
+
+typedef void (*pglogical_write_begin_fn)(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+typedef void (*pglogical_write_commit_fn)(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+typedef void (*pglogical_write_origin_fn)(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn);
+
+typedef void (*pglogical_write_insert_fn)(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+typedef void (*pglogical_write_update_fn)(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+typedef void (*pglogical_write_delete_fn)(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+typedef void (*write_startup_message_fn)(StringInfo out, List *msg);
+
+typedef struct PGLogicalProtoAPI
+{
+ pglogical_write_rel_fn write_rel;
+ pglogical_write_begin_fn write_begin;
+ pglogical_write_commit_fn write_commit;
+ pglogical_write_origin_fn write_origin;
+ pglogical_write_insert_fn write_insert;
+ pglogical_write_update_fn write_update;
+ pglogical_write_delete_fn write_delete;
+ write_startup_message_fn write_startup_message;
+} PGLogicalProtoAPI;
+
+
+typedef enum PGLogicalProtoType
+{
+ PGLogicalProtoNative,
+ PGLogicalProtoJson
+} PGLogicalProtoType;
+
+extern PGLogicalProtoAPI *pglogical_init_api(PGLogicalProtoType typ);
+
+#endif /* PG_LOGICAL_PROTO_H */
diff --git a/contrib/pglogical_output/pglogical_proto_json.c b/contrib/pglogical_output/pglogical_proto_json.c
new file mode 100644
index 0000000..ae5a591
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_json.c
@@ -0,0 +1,204 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_json.c
+ * pglogical protocol functions for json support
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_json.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto_json.h"
+
+#include "access/sysattr.h"
+#include "access/tuptoaster.h"
+#include "access/xact.h"
+
+#include "catalog/catversion.h"
+#include "catalog/index.h"
+
+#include "catalog/namespace.h"
+#include "catalog/pg_class.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_namespace.h"
+#include "catalog/pg_type.h"
+
+#include "commands/dbcommands.h"
+
+#include "executor/spi.h"
+
+#include "libpq/pqformat.h"
+
+#include "mb/pg_wchar.h"
+
+#ifdef HAVE_REPLICATION_ORIGINS
+#include "replication/origin.h"
+#endif
+
+#include "utils/builtins.h"
+#include "utils/json.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/typcache.h"
+
+
+/*
+ * Write BEGIN to the output stream.
+ */
+void
+pglogical_json_write_begin(StringInfo out, PGLogicalOutputData *data, ReorderBufferTXN *txn)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfoString(out, "\"action\":\"B\"");
+ appendStringInfo(out, ", \"has_catalog_changes\":\"%c\"",
+ txn->has_catalog_changes ? 't' : 'f');
+#ifdef HAVE_REPLICATION_ORIGINS
+ if (txn->origin_id != InvalidRepOriginId)
+ appendStringInfo(out, ", \"origin_id\":\"%u\"", txn->origin_id);
+#endif
+ if (!data->client_no_txinfo)
+ {
+ appendStringInfo(out, ", \"xid\":\"%u\"", txn->xid);
+ appendStringInfo(out, ", \"first_lsn\":\"%X/%X\"",
+ (uint32)(txn->first_lsn >> 32), (uint32)(txn->first_lsn));
+#ifdef HAVE_REPLICATION_ORIGINS
+ appendStringInfo(out, ", \"origin_lsn\":\"%X/%X\"",
+ (uint32)(txn->origin_lsn >> 32), (uint32)(txn->origin_lsn));
+#endif
+ if (txn->commit_time != 0)
+ appendStringInfo(out, ", \"commit_time\":\"%s\"",
+ timestamptz_to_str(txn->commit_time));
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write COMMIT to the output stream.
+ */
+void
+pglogical_json_write_commit(StringInfo out, PGLogicalOutputData *data, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfoString(out, "\"action\":\"C\"");
+ if (!data->client_no_txinfo)
+ {
+ appendStringInfo(out, ", \"final_lsn\":\"%X/%X\"",
+ (uint32)(txn->final_lsn >> 32), (uint32)(txn->final_lsn));
+ appendStringInfo(out, ", \"end_lsn\":\"%X/%X\"",
+ (uint32)(txn->end_lsn >> 32), (uint32)(txn->end_lsn));
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write a tuple to the outputstream, in the most efficient format possible.
+ */
+static void
+json_write_tuple(StringInfo out, Relation rel, HeapTuple tuple)
+{
+ TupleDesc desc;
+ Datum tupdatum,
+ json;
+
+ desc = RelationGetDescr(rel);
+ tupdatum = heap_copy_tuple_as_datum(tuple, desc);
+ json = DirectFunctionCall1(row_to_json, tupdatum);
+
+ appendStringInfoString(out, TextDatumGetCString(json));
+}
+
+/*
+ * Write change.
+ *
+ * Generic function handling DML changes.
+ */
+static void
+pglogical_json_write_change(StringInfo out, const char *change, Relation rel,
+ HeapTuple oldtuple, HeapTuple newtuple)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfo(out, "\"action\":\"%s\",\"relation\":[\"%s\",\"%s\"]",
+ change,
+ get_namespace_name(RelationGetNamespace(rel)),
+ RelationGetRelationName(rel));
+
+ if (oldtuple)
+ {
+ appendStringInfoString(out, ",\"oldtuple\":");
+ json_write_tuple(out, rel, oldtuple);
+ }
+ if (newtuple)
+ {
+ appendStringInfoString(out, ",\"newtuple\":");
+ json_write_tuple(out, rel, newtuple);
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write INSERT to the output stream.
+ */
+void
+pglogical_json_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple)
+{
+ pglogical_json_write_change(out, "I", rel, NULL, newtuple);
+}
+
+/*
+ * Write UPDATE to the output stream.
+ */
+void
+pglogical_json_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple)
+{
+ pglogical_json_write_change(out, "U", rel, oldtuple, newtuple);
+}
+
+/*
+ * Write DELETE to the output stream.
+ */
+void
+pglogical_json_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple)
+{
+ pglogical_json_write_change(out, "D", rel, oldtuple, NULL);
+}
+
+/*
+ * The startup message should be constructed as a json object, one
+ * key/value per DefElem list member.
+ */
+void
+json_write_startup_message(StringInfo out, List *msg)
+{
+ ListCell *lc;
+ bool first = true;
+
+ appendStringInfoString(out, "{\"action\":\"S\", \"params\": {");
+ foreach (lc, msg)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ if (first)
+ first = false;
+ else
+ appendStringInfoChar(out, ',');
+ escape_json(out, param->defname);
+ appendStringInfoChar(out, ':');
+ escape_json(out, strVal(param->arg));
+ }
+ appendStringInfoString(out, "}}");
+}
diff --git a/contrib/pglogical_output/pglogical_proto_json.h b/contrib/pglogical_output/pglogical_proto_json.h
new file mode 100644
index 0000000..d853e9e
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_json.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_json.h
+ * pglogical protocol, json implementation
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_json.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_JSON_H
+#define PG_LOGICAL_PROTO_JSON_H
+
+
+extern void pglogical_json_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+extern void pglogical_json_write_commit(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+extern void pglogical_json_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+extern void pglogical_json_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+extern void pglogical_json_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+extern void json_write_startup_message(StringInfo out, List *msg);
+
+#endif /* PG_LOGICAL_PROTO_JSON_H */
diff --git a/contrib/pglogical_output/pglogical_proto_native.c b/contrib/pglogical_output/pglogical_proto_native.c
new file mode 100644
index 0000000..baaf324
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_native.c
@@ -0,0 +1,494 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_native.c
+ * pglogical binary protocol functions
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_native.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto_native.h"
+
+#include "access/sysattr.h"
+#include "access/tuptoaster.h"
+#include "access/xact.h"
+
+#include "catalog/catversion.h"
+#include "catalog/index.h"
+
+#include "catalog/namespace.h"
+#include "catalog/pg_class.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_namespace.h"
+#include "catalog/pg_type.h"
+
+#include "commands/dbcommands.h"
+
+#include "executor/spi.h"
+
+#include "libpq/pqformat.h"
+
+#include "mb/pg_wchar.h"
+
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/typcache.h"
+
+#define IS_REPLICA_IDENTITY 1
+
+static void pglogical_write_attrs(StringInfo out, Relation rel);
+static void pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple);
+static char decide_datum_transfer(Form_pg_attribute att,
+ Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes);
+
+/*
+ * Write relation description to the output stream.
+ */
+void
+pglogical_write_rel(StringInfo out, Relation rel)
+{
+ const char *nspname;
+ uint8 nspnamelen;
+ const char *relname;
+ uint8 relnamelen;
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'R'); /* sending RELATION */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ nspname = get_namespace_name(rel->rd_rel->relnamespace);
+ if (nspname == NULL)
+ elog(ERROR, "cache lookup failed for namespace %u",
+ rel->rd_rel->relnamespace);
+ nspnamelen = strlen(nspname) + 1;
+
+ relname = NameStr(rel->rd_rel->relname);
+ relnamelen = strlen(relname) + 1;
+
+ pq_sendbyte(out, nspnamelen); /* schema name length */
+ pq_sendbytes(out, nspname, nspnamelen);
+
+ pq_sendbyte(out, relnamelen); /* table name length */
+ pq_sendbytes(out, relname, relnamelen);
+
+ /* send the attribute info */
+ pglogical_write_attrs(out, rel);
+}
+
+/*
+ * Write relation attributes to the outputstream.
+ */
+static void
+pglogical_write_attrs(StringInfo out, Relation rel)
+{
+ TupleDesc desc;
+ int i;
+ uint16 nliveatts = 0;
+ Bitmapset *idattrs;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'A'); /* sending ATTRS */
+
+ /* send number of live attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* fetch bitmap of REPLICATION IDENTITY attributes */
+ idattrs = RelationGetIndexAttrBitmap(rel, INDEX_ATTR_BITMAP_IDENTITY_KEY);
+
+ /* send the attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ Form_pg_attribute att = desc->attrs[i];
+ uint8 flags = 0;
+ uint16 len;
+ const char *attname;
+
+ if (att->attisdropped)
+ continue;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ idattrs))
+ flags |= IS_REPLICA_IDENTITY;
+
+ pq_sendbyte(out, 'C'); /* column definition follows */
+ pq_sendbyte(out, flags);
+
+ pq_sendbyte(out, 'N'); /* column name block follows */
+ attname = NameStr(att->attname);
+ len = strlen(attname) + 1;
+ pq_sendint(out, len, 2);
+ pq_sendbytes(out, attname, len); /* data */
+ }
+}
+
+/*
+ * Write BEGIN to the output stream.
+ */
+void
+pglogical_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'B'); /* BEGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, txn->final_lsn);
+ pq_sendint64(out, txn->commit_time);
+ pq_sendint(out, txn->xid, 4);
+}
+
+/*
+ * Write COMMIT to the output stream.
+ */
+void
+pglogical_write_commit(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'C'); /* sending COMMIT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* send fixed fields */
+ pq_sendint64(out, commit_lsn);
+ pq_sendint64(out, txn->end_lsn);
+ pq_sendint64(out, txn->commit_time);
+}
+
+/*
+ * Write ORIGIN to the output stream.
+ */
+void
+pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn)
+{
+ uint8 flags = 0;
+ uint8 len;
+
+ Assert(strlen(origin) < 255);
+
+ pq_sendbyte(out, 'O'); /* ORIGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, origin_lsn);
+
+ /* origin */
+ len = strlen(origin) + 1;
+ pq_sendbyte(out, len);
+ pq_sendbytes(out, origin, len);
+}
+
+/*
+ * Write INSERT to the output stream.
+ */
+void
+pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'I'); /* action INSERT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write UPDATE to the output stream.
+ */
+void
+pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'U'); /* action UPDATE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ if (oldtuple != NULL)
+ {
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+ }
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write DELETE to the output stream.
+ */
+void
+pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'D'); /* action DELETE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+}
+
+/*
+ * Most of the brains for startup message creation lives in
+ * pglogical_config.c, so this presently just sends the set of key/value pairs.
+ */
+void
+write_startup_message(StringInfo out, List *msg)
+{
+ ListCell *lc;
+
+ pq_sendbyte(out, 'S'); /* message type field */
+ pq_sendbyte(out, 1); /* startup message version */
+ foreach (lc, msg)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ /* null-terminated key and value pairs, in client_encoding */
+ pq_sendstring(out, param->defname);
+ pq_sendstring(out, strVal(param->arg));
+ }
+}
+
+/*
+ * Write a tuple to the outputstream, in the most efficient format possible.
+ */
+static void
+pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple)
+{
+ TupleDesc desc;
+ Datum values[MaxTupleAttributeNumber];
+ bool isnull[MaxTupleAttributeNumber];
+ int i;
+ uint16 nliveatts = 0;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'T'); /* sending TUPLE */
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* try to allocate enough memory from the get go */
+ enlargeStringInfo(out, tuple->t_len +
+ nliveatts * (1 + 4));
+
+ /*
+ * XXX: should this prove to be a relevant bottleneck, it might be
+ * interesting to inline heap_deform_tuple() here, we don't actually need
+ * the information in the form we get from it.
+ */
+ heap_deform_tuple(tuple, desc, values, isnull);
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ HeapTuple typtup;
+ Form_pg_type typclass;
+ Form_pg_attribute att = desc->attrs[i];
+ char transfer_type;
+
+ /* skip dropped columns */
+ if (att->attisdropped)
+ continue;
+
+ if (isnull[i])
+ {
+ pq_sendbyte(out, 'n'); /* null column */
+ continue;
+ }
+ else if (att->attlen == -1 && VARATT_IS_EXTERNAL_ONDISK(values[i]))
+ {
+ pq_sendbyte(out, 'u'); /* unchanged toast column */
+ continue;
+ }
+
+ typtup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid));
+ if (!HeapTupleIsValid(typtup))
+ elog(ERROR, "cache lookup failed for type %u", att->atttypid);
+ typclass = (Form_pg_type) GETSTRUCT(typtup);
+
+ transfer_type = decide_datum_transfer(att, typclass,
+ data->allow_internal_basetypes,
+ data->allow_binary_basetypes);
+
+ switch (transfer_type)
+ {
+ case 'i':
+ pq_sendbyte(out, 'i'); /* internal-format binary data follows */
+
+ /* pass by value */
+ if (att->attbyval)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ enlargeStringInfo(out, att->attlen);
+ store_att_byval(out->data + out->len, values[i],
+ att->attlen);
+ out->len += att->attlen;
+ out->data[out->len] = '\0';
+ }
+ /* fixed length non-varlena pass-by-reference type */
+ else if (att->attlen > 0)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ appendBinaryStringInfo(out, DatumGetPointer(values[i]),
+ att->attlen);
+ }
+ /* varlena type */
+ else if (att->attlen == -1)
+ {
+ char *data = DatumGetPointer(values[i]);
+
+ /* send indirect datums inline */
+ if (VARATT_IS_EXTERNAL_INDIRECT(values[i]))
+ {
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, data);
+ data = (char *) redirect.pointer;
+ }
+
+ Assert(!VARATT_IS_EXTERNAL(data));
+
+ pq_sendint(out, VARSIZE_ANY(data), 4); /* length */
+
+ appendBinaryStringInfo(out, data, VARSIZE_ANY(data));
+ }
+ else
+ elog(ERROR, "unsupported tuple type");
+
+ break;
+
+ case 'b':
+ {
+ bytea *outputbytes;
+ int len;
+
+ pq_sendbyte(out, 'b'); /* binary send/recv data follows */
+
+ outputbytes = OidSendFunctionCall(typclass->typsend,
+ values[i]);
+
+ len = VARSIZE(outputbytes) - VARHDRSZ;
+ pq_sendint(out, len, 4); /* length */
+ pq_sendbytes(out, VARDATA(outputbytes), len); /* data */
+ pfree(outputbytes);
+ }
+ break;
+
+ default:
+ {
+ char *outputstr;
+ int len;
+
+ pq_sendbyte(out, 't'); /* 'text' data follows */
+
+ outputstr = OidOutputFunctionCall(typclass->typoutput,
+ values[i]);
+ len = strlen(outputstr) + 1;
+ pq_sendint(out, len, 4); /* length */
+ appendBinaryStringInfo(out, outputstr, len); /* data */
+ pfree(outputstr);
+ }
+ }
+
+ ReleaseSysCache(typtup);
+ }
+}
+
+/*
+ * Make the executive decision about which protocol to use.
+ */
+static char
+decide_datum_transfer(Form_pg_attribute att, Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes)
+{
+ /*
+ * Use the binary protocol, if allowed, for builtin & plain datatypes.
+ */
+ if (allow_internal_basetypes &&
+ typclass->typtype == 'b' &&
+ att->atttypid < FirstNormalObjectId &&
+ typclass->typelem == InvalidOid)
+ {
+ return 'i';
+ }
+ /*
+ * Use send/recv, if allowed, if the type is plain or builtin.
+ *
+ * XXX: we can't use send/recv for array or composite types for now due to
+ * the embedded oids.
+ */
+ else if (allow_binary_basetypes &&
+ OidIsValid(typclass->typreceive) &&
+ (att->atttypid < FirstNormalObjectId || typclass->typtype != 'c') &&
+ (att->atttypid < FirstNormalObjectId || typclass->typelem == InvalidOid))
+ {
+ return 'b';
+ }
+
+ return 't';
+}
diff --git a/contrib/pglogical_output/pglogical_proto_native.h b/contrib/pglogical_output/pglogical_proto_native.h
new file mode 100644
index 0000000..729bee0
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_native.h
@@ -0,0 +1,37 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_native.h
+ * pglogical protocol, native implementation
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_native.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_NATIVE_H
+#define PG_LOGICAL_PROTO_NATIVE_H
+
+
+extern void pglogical_write_rel(StringInfo out, Relation rel);
+
+extern void pglogical_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+extern void pglogical_write_commit(StringInfo out,PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+extern void pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn);
+
+extern void pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+extern void pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+extern void pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+extern void write_startup_message(StringInfo out, List *msg);
+
+#endif /* PG_LOGICAL_PROTO_NATIVE_H */
diff --git a/contrib/pglogical_output/regression.conf b/contrib/pglogical_output/regression.conf
new file mode 100644
index 0000000..367f706
--- /dev/null
+++ b/contrib/pglogical_output/regression.conf
@@ -0,0 +1,2 @@
+wal_level = logical
+max_replication_slots = 4
diff --git a/contrib/pglogical_output/sql/basic_json.sql b/contrib/pglogical_output/sql/basic_json.sql
new file mode 100644
index 0000000..e8a2352
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_json.sql
@@ -0,0 +1,24 @@
+\i sql/basic_setup.sql
+
+-- Simple decode with text-format tuples
+TRUNCATE TABLE json_decoding_output;
+
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+SELECT * FROM get_startup_params();
+SELECT * FROM get_queued_data();
+
+TRUNCATE TABLE json_decoding_output;
+
+\i sql/basic_teardown.sql
diff --git a/contrib/pglogical_output/sql/basic_native.sql b/contrib/pglogical_output/sql/basic_native.sql
new file mode 100644
index 0000000..5b6ec4b
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_native.sql
@@ -0,0 +1,27 @@
+\i sql/basic_setup.sql
+
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+
+\i sql/basic_teardown.sql
diff --git a/contrib/pglogical_output/sql/basic_setup.sql b/contrib/pglogical_output/sql/basic_setup.sql
new file mode 100644
index 0000000..19e154c
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_setup.sql
@@ -0,0 +1,62 @@
+SET synchronous_commit = on;
+
+-- Schema setup
+
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Queue up some work to decode with a variety of types
+
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+
+
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
diff --git a/contrib/pglogical_output/sql/basic_teardown.sql b/contrib/pglogical_output/sql/basic_teardown.sql
new file mode 100644
index 0000000..d7a752f
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_teardown.sql
@@ -0,0 +1,4 @@
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/sql/cleanup.sql b/contrib/pglogical_output/sql/cleanup.sql
new file mode 100644
index 0000000..e7a02c8
--- /dev/null
+++ b/contrib/pglogical_output/sql/cleanup.sql
@@ -0,0 +1,4 @@
+DROP TABLE excluded_startup_keys;
+DROP TABLE json_decoding_output;
+DROP FUNCTION get_queued_data();
+DROP FUNCTION get_startup_params();
diff --git a/contrib/pglogical_output/sql/encoding_json.sql b/contrib/pglogical_output/sql/encoding_json.sql
new file mode 100644
index 0000000..543c306
--- /dev/null
+++ b/contrib/pglogical_output/sql/encoding_json.sql
@@ -0,0 +1,58 @@
+SET synchronous_commit = on;
+
+-- This file doesn't share common setup with the native tests,
+-- since it's specific to how the text protocol handles encodings.
+
+CREATE TABLE enctest(blah text);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+
+SET client_encoding = 'UTF-8';
+INSERT INTO enctest(blah)
+VALUES
+('áàä'),('fl'), ('½⅓'), ('カンジ');
+RESET client_encoding;
+
+
+SET client_encoding = 'LATIN-1';
+
+-- Will ERROR, explicit encoding request doesn't match client_encoding
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+-- Will succeed since we don't request any encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+-- Will succeed since it matches the current encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'LATIN-1',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+RESET client_encoding;
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE enctest;
diff --git a/contrib/pglogical_output/sql/hooks_json.sql b/contrib/pglogical_output/sql/hooks_json.sql
new file mode 100644
index 0000000..cd58960
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_json.sql
@@ -0,0 +1,49 @@
+\i sql/hooks_setup.sql
+
+
+-- Test table filter
+TRUNCATE TABLE json_decoding_output;
+
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+SELECT * FROM get_startup_params();
+SELECT * FROM get_queued_data();
+
+-- test action filter
+TRUNCATE TABLE json_decoding_output;
+
+INSERT INTO json_decoding_output (ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+SELECT * FROM get_startup_params();
+SELECT * FROM get_queued_data();
+
+TRUNCATE TABLE json_decoding_output;
+
+\i sql/hooks_teardown.sql
diff --git a/contrib/pglogical_output/sql/hooks_native.sql b/contrib/pglogical_output/sql/hooks_native.sql
new file mode 100644
index 0000000..e2bfc54
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_native.sql
@@ -0,0 +1,48 @@
+\i sql/hooks_setup.sql
+
+-- Regular hook setup
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+
+-- Test action filter
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+
+-- Invalid row fiter hook function
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+
+-- Hook filter functoin with wrong signature
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+
+\i sql/hooks_teardown.sql
diff --git a/contrib/pglogical_output/sql/hooks_setup.sql b/contrib/pglogical_output/sql/hooks_setup.sql
new file mode 100644
index 0000000..4de15b7
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_setup.sql
@@ -0,0 +1,37 @@
+CREATE EXTENSION pglogical_output_plhooks;
+
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
diff --git a/contrib/pglogical_output/sql/hooks_teardown.sql b/contrib/pglogical_output/sql/hooks_teardown.sql
new file mode 100644
index 0000000..837e2d0
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_teardown.sql
@@ -0,0 +1,10 @@
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/sql/params_native.sql b/contrib/pglogical_output/sql/params_native.sql
new file mode 100644
index 0000000..8b08732
--- /dev/null
+++ b/contrib/pglogical_output/sql/params_native.sql
@@ -0,0 +1,95 @@
+SET synchronous_commit = on;
+
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+--
+-- Various invalid parameter combos:
+--
+
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
diff --git a/contrib/pglogical_output/sql/prep.sql b/contrib/pglogical_output/sql/prep.sql
new file mode 100644
index 0000000..26e79c8
--- /dev/null
+++ b/contrib/pglogical_output/sql/prep.sql
@@ -0,0 +1,30 @@
+CREATE TABLE excluded_startup_keys (key_name text primary key);
+
+INSERT INTO excluded_startup_keys
+VALUES
+('pg_version_num'),('pg_version'),('pg_catversion'),('binary.basetypes_major_version'),('binary.integer_datetimes'),('binary.bigendian'),('binary.maxalign'),('binary.binary_pg_version'),('sizeof_int'),('sizeof_long'),('sizeof_datum');
+
+CREATE UNLOGGED TABLE json_decoding_output(ch jsonb, rn integer);
+
+CREATE OR REPLACE FUNCTION get_startup_params()
+RETURNS TABLE ("key" text, "value" jsonb)
+LANGUAGE sql
+AS $$
+SELECT key, value
+FROM json_decoding_output
+CROSS JOIN LATERAL jsonb_each(ch -> 'params')
+WHERE rn = 1
+ AND key NOT IN (SELECT * FROM excluded_startup_keys)
+ AND ch ->> 'action' = 'S'
+ORDER BY key;
+$$;
+
+CREATE OR REPLACE FUNCTION get_queued_data()
+RETURNS TABLE (data jsonb)
+LANGUAGE sql
+AS $$
+SELECT ch
+FROM json_decoding_output
+WHERE rn > 1
+ORDER BY rn ASC;
+$$;
diff --git a/contrib/pglogical_output_plhooks/.gitignore b/contrib/pglogical_output_plhooks/.gitignore
new file mode 100644
index 0000000..140f8cf
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/.gitignore
@@ -0,0 +1 @@
+*.so
diff --git a/contrib/pglogical_output_plhooks/Makefile b/contrib/pglogical_output_plhooks/Makefile
new file mode 100644
index 0000000..ecd3f89
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/Makefile
@@ -0,0 +1,13 @@
+MODULES = pglogical_output_plhooks
+EXTENSION = pglogical_output_plhooks
+DATA = pglogical_output_plhooks--1.0.sql
+DOCS = README.pglogical_output_plhooks
+
+subdir = contrib/pglogical_output_plhooks
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# Allow the hook plugin to see the pglogical_output headers
+# Necessary because !PGXS builds don't respect PG_CPPFLAGS
+override CPPFLAGS := $(CPPFLAGS) -I$(top_srcdir)/contrib/pglogical_output
diff --git a/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks b/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
new file mode 100644
index 0000000..f2ad9d4
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
@@ -0,0 +1,158 @@
+pglogical_output_plhooks is an example module for pglogical_output, showing how
+hooks can be implemented.
+
+It provides C wrappers to allow hooks to be written in any supported PL,
+such as PL/PgSQL.
+
+No effort is made to be efficient. To avoid the need to set up cache
+invalidation handling function calls are done via oid each time, with no
+FmgrInfo caching. Also, memory contexts are reset rather freely. If you
+want efficiency, write your hook in C.
+
+(Catalog timetravel is another reason not to write hooks in PLs; see below).
+
+Simple pointless example
+===
+
+To compile and install, just "make USE_PGXS=1 install". Note that pglogical
+must already be installed so that its headers can be found. You might have
+to set the `PATH` so that `pg_config` can be found.
+
+To use it:
+
+ CREATE EXTENSION pglogical_output_plhooks IN SCHEMA public;
+
+in the target database.
+
+Then create at least one hook procedure, of the supported hooks listed below.
+For the sake of this example we'll use some of the toy examples provided in the
+extension:
+
+* startup function: pglo_plhooks_demo_startup
+* row filter: pglo_plhooks_demo_row_filter
+* txn filter: pglo_plhooks_demo_txn_filter
+* shutdown function: pglo_plhooks_demo_shutdown
+
+Now add some arguments to your pglogical_output client's logical decoding setup
+parameters to specify the hook setup function and to tell
+pglogical_output_plhooks about one or more of the hooks you wish it to run. For
+example you might add the following parameters:
+
+ hooks.setup_function, public.pglo_plhooks_setup_fn,
+ pglo_plhooks.startup_hook, pglo_plhooks_demo_startup,
+ pglo_plhooks.row_filter_hook, pglo_plhooks_demo_row_filter,
+ pglo_plhooks.txn_filter_hook, pglo_plhooks_demo_txn_filter,
+ pglo_plhooks.shutdown_hook, pglo_plhooks_demo_shutdown,
+ pglo_plhooks.client_hook_arg, 'whatever-you-want'
+
+to configure the extension to load its hooks, then configure all the demo hooks.
+
+Why the preference for C hooks?
+===
+
+Speed. The row filter hook is called for *every single row* replicated.
+
+If a hook raises an ERROR then replication will probably stop. You won't be
+able to fix it either, because when you change the hook definition the new
+definition won't be visible in the catalogs at the current replay position due
+to catalog time travel. The old definition that raises an error will keep being
+used. You'll need to remove the problem hook from your logical decoding startup
+parameters, which will disable use the hook entirely, until replay proceeds
+past the point you fixed the problem with the hook function.
+
+Similarly, if you try to add use of a newly defined hook on an existing
+replication slot that hasn't replayed past the point you defined the hook yet,
+you'll get an error complaining that the hook function doesn't exist. Even
+though it clearly does when you look at it in psql. The reason is the same: in
+the time traveled catalogs it really doesn't exist. You have to replay past the
+point the hook was created then enable it. In this case the
+pglogical_output_plhooks startup hook will actually see your functions, but
+fail when it tries to call them during decoding since they'll appear to have
+vanished.
+
+If you write your hooks in C you can redefine them rather more easily, since
+the function definition is not subject to catalog timetravel. More importantly,
+it'll probably be a lot faster. The plhooks code has to do a lot of translation
+to pass information to the PL functions and more to get results back; it also
+has to do a lot of memory allocations and a memory context reset after each
+call. That all adds up.
+
+(You could actually write C functions to be called by this extension, but
+that'd be crazy.)
+
+Available hooks
+===
+
+The four hooks provided by pglogical_output are exposed by the module. See the
+pglogical_output documentation for details on what each hook does and when it
+runs.
+
+A function for each hook must have *exactly* the specified parameters and
+return value, or you'll get an error.
+
+None of the functions may return NULL. If they do you'll get an error.
+
+If you specified `pglo_plhooks.client_hook_arg` in the startup parameters it is
+passed as `client_hook_arg` to all hooks. If not specified the empty string is
+passed.
+
+You can find some toy examples in `pglogical_output_plhooks--1.0.sql`.
+
+
+
+Startup hook
+---
+
+Configured with `pglo_plhooks.startup_hook` startup parameter. Runs when
+logical decoding starts.
+
+Signature *must* be:
+
+ CREATE FUNCTION whatever_funcname(startup_params text[], client_hook_arg text)
+ RETURNS text[]
+
+startup_params is an array of the startup params passed to the pglogical output
+plugin, as alternating key/value elements in text representation.
+
+client_hook_arg is also passed.
+
+The return value is an array of alternating key/value elements forming a set
+of parameters you wish to add to the startup reply message sent by pglogical
+on decoding start. It must not be null; return `ARRAY[]::text[]` if you don't
+want to add any params.
+
+Transaction filter
+---
+
+The arguments are the replication origin identifier and the client hook param.
+
+The return value is true to keep the transaction, false to discard it.
+
+Signature:
+
+ CREATE FUNCTION whatevername(origin_id int, client_hook_arg text)
+ RETURNS boolean
+
+Row filter
+--
+
+Called for each row. Return true to replicate the row, false to discard it.
+
+Arguments are the oid of the affected relation, and the change type: 'I'nsert,
+'U'pdate or 'D'elete. There is no way to access the change data - columns changed,
+new values, etc.
+
+Signature:
+
+ CREATE FUNCTION whatevername(affected_rel regclass, change_type "char", client_hook_arg text)
+ RETURNS boolean
+
+Shutdown hook
+--
+
+Pretty uninteresting, but included for completeness.
+
+Signature:
+
+ CREATE FUNCTION whatevername(client_hook_arg text)
+ RETURNS void
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql b/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
new file mode 100644
index 0000000..cdd2af3
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
@@ -0,0 +1,89 @@
+\echo Use "CREATE EXTENSION pglogical_output_plhooks" to load this file. \quit
+
+-- Use @extschema@ or leave search_path unchanged, don't use explicit schema
+
+CREATE FUNCTION pglo_plhooks_setup_fn(internal)
+RETURNS void
+STABLE
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+COMMENT ON FUNCTION pglo_plhooks_setup_fn(internal)
+IS 'Register pglogical output pl hooks. See docs for how to specify functions';
+
+--
+-- Called as the startup hook.
+--
+-- There's no useful way to expose the private data segment, so you
+-- just don't get to use that from pl hooks at this point. The C
+-- wrapper will extract a startup param named pglo_plhooks.client_hook_arg
+-- for you and pass it as client_hook_arg to all callbacks, though.
+--
+-- For implementation convenience, a null client_hook_arg is passed
+-- as the empty string.
+--
+-- Must return the empty array, not NULL, if it has nothing to add.
+--
+CREATE FUNCTION pglo_plhooks_demo_startup(startup_params text[], client_hook_arg text)
+RETURNS text[]
+LANGUAGE plpgsql AS $$
+DECLARE
+ elem text;
+ paramname text;
+ paramvalue text;
+BEGIN
+ FOREACH elem IN ARRAY startup_params
+ LOOP
+ IF elem IS NULL THEN
+ RAISE EXCEPTION 'Startup params may not be null';
+ END IF;
+
+ IF paramname IS NULL THEN
+ paramname := elem;
+ ELSIF paramvalue IS NULL THEN
+ paramvalue := elem;
+ ELSE
+ RAISE NOTICE 'got param: % = %', paramname, paramvalue;
+ paramname := NULL;
+ paramvalue := NULL;
+ END IF;
+ END LOOP;
+
+ RETURN ARRAY['pglo_plhooks_demo_startup_ran', 'true', 'otherparam', '42'];
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_txn_filter(origin_id int, client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- Not much to filter on really...
+ RAISE NOTICE 'Got tx with origin %',origin_id;
+ RETURN true;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_row_filter(affected_rel regclass, change_type "char", client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- This is a totally absurd test, since it checks if the upstream user
+ -- doing replication has rights to make modifications that have already
+ -- been committed and are being decoded for replication. Still, it shows
+ -- how the hook works...
+ IF pg_catalog.has_table_privilege(current_user, affected_rel,
+ CASE change_type WHEN 'I' THEN 'INSERT' WHEN 'U' THEN 'UPDATE' WHEN 'D' THEN 'DELETE' END)
+ THEN
+ RETURN true;
+ ELSE
+ RETURN false;
+ END IF;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_shutdown(client_hook_arg text)
+RETURNS void
+LANGUAGE plpgsql AS $$
+BEGIN
+ RAISE NOTICE 'Decoding shutdown';
+END;
+$$
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
new file mode 100644
index 0000000..a5144f0
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
@@ -0,0 +1,414 @@
+#include "postgres.h"
+
+#include "pglogical_output/hooks.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_type.h"
+
+#include "nodes/makefuncs.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/reorderbuffer.h"
+
+#include "utils/acl.h"
+#include "utils/array.h"
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+
+#include "fmgr.h"
+#include "miscadmin.h"
+
+PG_MODULE_MAGIC;
+
+PGDLLEXPORT extern Datum pglo_plhooks_setup_fn(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglo_plhooks_setup_fn);
+
+void pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args);
+void pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args);
+bool pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args);
+bool pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args);
+
+typedef struct PLHPrivate
+{
+ const char *client_arg;
+ Oid startup_hook;
+ Oid shutdown_hook;
+ Oid row_filter_hook;
+ Oid txn_filter_hook;
+ MemoryContext hook_call_context;
+} PLHPrivate;
+
+static void read_parameters(PLHPrivate *private, List *in_params);
+static Oid find_startup_hook(const char *proname);
+static Oid find_shutdown_hook(const char *proname);
+static Oid find_row_filter_hook(const char *proname);
+static Oid find_txn_filter_hook(const char *proname);
+static void exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params);
+
+void
+pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args)
+{
+ PLHPrivate *private;
+
+ /* pglogical_output promises to call us in a tx */
+ Assert(IsTransactionState());
+
+ /* Allocated in hook memory context, scoped to the logical decoding session: */
+ startup_args->private_data = private = (PLHPrivate*)palloc(sizeof(PLHPrivate));
+
+ private->startup_hook = InvalidOid;
+ private->shutdown_hook = InvalidOid;
+ private->row_filter_hook = InvalidOid;
+ private->txn_filter_hook = InvalidOid;
+ /* client_arg is the empty string when not specified to simplify function calls */
+ private->client_arg = "";
+
+ read_parameters(private, startup_args->in_params);
+
+ private->hook_call_context = AllocSetContextCreate(CurrentMemoryContext,
+ "pglogical_output plhooks hook call context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+
+ if (private->startup_hook != InvalidOid)
+ exec_user_startup_hook(private, startup_args->in_params, &startup_args->out_params);
+}
+
+void
+pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args)
+{
+ PLHPrivate *private = (PLHPrivate*)shutdown_args->private_data;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->shutdown_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo shutdown hook with %s", private->client_arg);
+ (void) OidFunctionCall1(
+ private->shutdown_hook,
+ CStringGetTextDatum(private->client_arg));
+ elog(DEBUG3, "called pglo shutdown hook");
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+}
+
+bool
+pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)rowfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->row_filter_hook))
+ {
+ char change_type;
+ switch (rowfilter_args->change_type)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ change_type = 'I';
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ change_type = 'U';
+ break;
+ case REORDER_BUFFER_CHANGE_DELETE:
+ change_type = 'D';
+ break;
+ default:
+ elog(ERROR, "unknown change type %d", rowfilter_args->change_type);
+ change_type = '0'; /* silence compiler */
+ }
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo row filter hook with (%u,%c,%s)",
+ rowfilter_args->changed_rel->rd_id, change_type,
+ private->client_arg);
+ ret = DatumGetBool(OidFunctionCall3(
+ private->row_filter_hook,
+ ObjectIdGetDatum(rowfilter_args->changed_rel->rd_id),
+ CharGetDatum(change_type),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "called pglo row filter hook, returns %d", (int)ret);
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+bool
+pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)txnfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+
+ if (OidIsValid(private->txn_filter_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ elog(DEBUG3, "calling pglo txn filter hook with (%hu,%s)",
+ txnfilter_args->origin_id, private->client_arg);
+ ret = DatumGetBool(OidFunctionCall2(
+ private->txn_filter_hook,
+ UInt16GetDatum(txnfilter_args->origin_id),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "calling pglo txn filter hook, returns %d", (int)ret);
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+Datum
+pglo_plhooks_setup_fn(PG_FUNCTION_ARGS)
+{
+ struct PGLogicalHooks *hooks = (struct PGLogicalHooks*) PG_GETARG_POINTER(0);
+
+ /* Your code doesn't need this, it's just for the tests: */
+ Assert(hooks != NULL);
+ Assert(hooks->hooks_private_data == NULL);
+ Assert(hooks->startup_hook == NULL);
+ Assert(hooks->shutdown_hook == NULL);
+ Assert(hooks->row_filter_hook == NULL);
+ Assert(hooks->txn_filter_hook == NULL);
+
+ /*
+ * Just assign the hook pointers. We're not meant to do much
+ * work here.
+ *
+ * Note that private_data is left untouched, to be set up by the
+ * startup hook.
+ */
+ hooks->startup_hook = pglo_plhooks_startup;
+ hooks->shutdown_hook = pglo_plhooks_shutdown;
+ hooks->row_filter_hook = pglo_plhooks_row_filter;
+ hooks->txn_filter_hook = pglo_plhooks_txn_filter;
+ elog(DEBUG3, "configured pglo hooks");
+
+ PG_RETURN_VOID();
+}
+
+static void
+exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params)
+{
+ ArrayType *startup_params;
+ Datum ret;
+ ListCell *lc;
+ Datum *startup_params_elems;
+ bool *startup_params_isnulls;
+ int n_startup_params;
+ int i;
+ MemoryContext old_ctx;
+
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ /*
+ * Build the input parameter array. NULL parameters are passed as the
+ * empty string for the sake of convenience. Each param is two
+ * elements, a key then a value element.
+ */
+ n_startup_params = list_length(in_params) * 2;
+ startup_params_elems = (Datum*)palloc0(sizeof(Datum)*n_startup_params);
+
+ i = 0;
+ foreach (lc, in_params)
+ {
+ DefElem * elem = (DefElem*)lfirst(lc);
+ const char *val;
+
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ val = "";
+ else
+ val = strVal(elem->arg);
+
+ startup_params_elems[i++] = CStringGetTextDatum(elem->defname);
+ startup_params_elems[i++] = CStringGetTextDatum(val);
+ }
+ Assert(i == n_startup_params);
+
+ startup_params = construct_array(startup_params_elems, n_startup_params,
+ TEXTOID, -1, false, 'i');
+
+ ret = OidFunctionCall2(
+ private->startup_hook,
+ PointerGetDatum(startup_params),
+ CStringGetTextDatum(private->client_arg));
+
+ /*
+ * deconstruct return array and add pairs of results to a DefElem list.
+ */
+ deconstruct_array(DatumGetArrayTypeP(ret), TEXTARRAYOID,
+ -1, false, 'i', &startup_params_elems, &startup_params_isnulls,
+ &n_startup_params);
+
+
+ *out_params = NIL;
+ for (i = 0; i < n_startup_params; i = i + 2)
+ {
+ char *value;
+ DefElem *elem;
+
+ if (startup_params_isnulls[i])
+ elog(ERROR, "Array entry corresponding to a key was null at idx=%d", i);
+
+ if (startup_params_isnulls[i+1])
+ value = "";
+ else
+ value = TextDatumGetCString(startup_params_elems[i+1]);
+
+ elem = makeDefElem(
+ TextDatumGetCString(startup_params_elems[i]),
+ (Node*)makeString(value));
+
+ *out_params = lcons(elem, *out_params);
+ }
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+}
+
+static void
+read_parameters(PLHPrivate *private, List *in_params)
+{
+ ListCell *option;
+
+ foreach(option, in_params)
+ {
+ DefElem *elem = lfirst(option);
+
+ if (pg_strcasecmp("pglo_plhooks.client_hook_arg", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.client_hook_arg may not be NULL");
+ private->client_arg = pstrdup(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.startup_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.startup_hook may not be NULL");
+ private->startup_hook = find_startup_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.shutdown_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.shutdown_hook may not be NULL");
+ private->shutdown_hook = find_shutdown_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.txn_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.txn_filter_hook may not be NULL");
+ private->txn_filter_hook = find_txn_filter_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.row_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.row_filter_hook may not be NULL");
+ private->row_filter_hook = find_row_filter_hook(strVal(elem->arg));
+ }
+ }
+}
+
+static Oid
+find_hook_fn(const char *funcname, Oid funcargtypes[], int nfuncargtypes, Oid returntype)
+{
+ Oid funcid;
+ List *qname;
+
+ qname = stringToQualifiedNameList(funcname);
+
+ /* find the the function */
+ funcid = LookupFuncName(qname, nfuncargtypes, funcargtypes, false);
+
+ /* Check expected return type */
+ if (get_func_rettype(funcid) != returntype)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s doesn't return expected type %d",
+ NameListToString(qname), returntype)));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(qname))));
+ }
+
+ list_free_deep(qname);
+
+ return funcid;
+}
+
+static Oid
+find_startup_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = TEXTARRAYOID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, VOIDOID);
+}
+
+static Oid
+find_shutdown_hook(const char *proname)
+{
+ Oid argtypes[1];
+
+ argtypes[0] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 1, VOIDOID);
+}
+
+static Oid
+find_row_filter_hook(const char *proname)
+{
+ Oid argtypes[3];
+
+ argtypes[0] = REGCLASSOID;
+ argtypes[1] = CHAROID;
+ argtypes[2] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 3, BOOLOID);
+}
+
+static Oid
+find_txn_filter_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = INT4OID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, BOOLOID);
+}
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
new file mode 100644
index 0000000..647b9ef
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
@@ -0,0 +1,4 @@
+comment = 'pglogical_output pl hooks'
+default_version = '1.0'
+module_pathname = '$libdir/pglogical_output_plhooks'
+relocatable = false
--
2.1.0
Hi,
I can't really do huge review considering I wrote half of the code, but
I have couple of things I noticed.
First, I wonder if it would be useful to mention somewhere, even if it's
only here in the mailing list how can the protocol be extended in
non-breaking way in future for transaction streaming if we ever get
that. I am mainly asking for this because the protocol does not
currently send xid for every change as it's not necessary, but for
streaming it will be. I know of couple of ways myself but I think they
should be described publicly.
The other thing is that I think we don't need the "forward_changesets"
parameter which currently decides if to forward changes that didn't
originate on local node. There is already hook for origin filtering
which provides same functionality in more flexible way so it seems
redundant to also have special boolean option for it.
--
Petr Jelinek http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 2 December 2015 at 18:38, Petr Jelinek <petr@2ndquadrant.com> wrote:
First, I wonder if it would be useful to mention somewhere, even if it's
only here in the mailing list how can the protocol be extended in
non-breaking way in future for transaction streaming if we ever get that.
Good point.
I'll address that in the DESIGN.md in the next rev.
Separately, it's looking like xact streaming is possibly more complex than
I hoped due to cache invalidation issues, but I haven't been able to fully
understand the problem yet.
The other thing is that I think we don't need the "forward_changesets"
parameter which currently decides if to forward changes that didn't
originate on local node. There is already hook for origin filtering which
provides same functionality in more flexible way so it seems redundant to
also have special boolean option for it.
Removed, change pushed.
Also pushed a change to expose the decoded row data to row filter hooks.
I won't cut a v4 for this, as I'm working on merging the SGML-ified docs
and will do a v4 with that and the above readme change once that's done.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
On Mon, Dec 7, 2015 at 12:10 PM, Craig Ringer <craig@2ndquadrant.com> wrote:
Removed, change pushed.
Also pushed a change to expose the decoded row data to row filter hooks.
I won't cut a v4 for this, as I'm working on merging the SGML-ified docs and
will do a v4 with that and the above readme change once that's done.
Patch is moved to next CF, you seem to be still working on it..
--
Michael
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 22 December 2015 at 15:17, Michael Paquier <michael.paquier@gmail.com>
wrote:
On Mon, Dec 7, 2015 at 12:10 PM, Craig Ringer <craig@2ndquadrant.com>
wrote:Removed, change pushed.
Also pushed a change to expose the decoded row data to row filter hooks.
I won't cut a v4 for this, as I'm working on merging the SGML-ified docs
and
will do a v4 with that and the above readme change once that's done.
Patch is moved to next CF, you seem to be still working on it.
Thanks.
Other than SGML-ified docs it's ready. The docs are a hard pre-req of
course. In any case most people appear to be waiting for the downstream
before looking at it at all, so bumping it makes sense.
I'm a touch frustrated by that, as a large part of the point of submitting
the output plugin separately and in advance of the downstream was to get
attention for it separately, as its own entity. A lot of effort has been
put into making this usable for more than just a data source for
pglogical's replication tools. In retrospect naming it pglogical_output was
probably unwise.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
On Tue, Dec 22, 2015 at 4:55 AM, Craig Ringer <craig@2ndquadrant.com> wrote:
I'm a touch frustrated by that, as a large part of the point of submitting
the output plugin separately and in advance of the downstream was to get
attention for it separately, as its own entity. A lot of effort has been put
into making this usable for more than just a data source for pglogical's
replication tools. In retrospect naming it pglogical_output was probably
unwise.
It's not too late to rename it.
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
The following review has been posted through the commitfest application:
make installcheck-world: tested, passed
Implements feature: not tested
Spec compliant: not tested
Documentation: tested, failed
Applies cleanly on current master (b416c0bb622ce5d33fdbec3bbce00451132f10ec).
Builds without any problems on current Debian unstable (am64 arch, GCC 5.3.1-4, glibc 2.21-6)
There are 2 errors in tests, but they also occur on trunk build.
parallel group (20 tests): json_encoding combocid portals_p2 advisory_lock tsdicts xmlmap equivclass guc functional_deps dependency json select_views cluster tsearch window jsonb indirect_toast bitmapops foreign_key foreign_data
select_views ... FAILED
portals_p2 ... ok
parallel group (2 tests): create_view create_index
create_index ... FAILED
create_view ... ok
README.md:
+Only one database is replicated, rather than the whole PostgreSQL install. A
[...]
+Unlike block-level ("physical") streaming replication, the change stream from
+the `pglogical` output plugin is compatible across different PostgreSQL
+versions and can even be consumed by non-PostgreSQL clients.
+
+Because logical decoding is used, only the changed rows are sent on the wire.
+There's no index change data, no vacuum activity, etc transmitted.
+
+The use of a replication slot means that the change stream is reliable and
+crash-safe. If
Isn't it feature of logical replication, not this particular plugin?
I'm not sure whether all that text needs to be repeated here.
OTOH this is good summary - so maybe just add links to base
documentation about replication, logical replication, slots, etc.
+# Usage
+
+The overall flow of client/server interaction is:
This part (flow) belongs to DESIGN.md, not to usage.
+* [Client issues `IDENTIFY_SYSTEM`
Is the [ needed here?
protocol.txt
Contains wrapped lines, but also very long lines. While long
lines make sense for tables, they should not occur in paragraphs, e.g. in
+== Arguments client supplies to output plugin
and following ones. It looks like parts of this file were formatted, and parts not.
In summary, documentation is mostly OK, and it describe plugin quite nicely.
The only thing I haven't fully understood is COPY-related - so it might be
good to extend a bit. And how COPY relates to JSON output format?
pglogical_output_plhooks.c
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
Is it needed? I mean - will tris module compiled for 9.4 (or earlier)
versions, or will it be 9.5 (or even 9.6)+ only?
pglogical_output.c
+ /*
+ * Will we forward changesets? We have to if we're on 9.4;
+ * otherwise honour the client's request.
+ */
+ if (PG_VERSION_NUM/100 == 904)
+ {
+ /*
+ * 9.4 unconditionally forwards changesets due to lack of
+ * replication origins, and it can't ever send origin info
+ * for the same reason.
+ */
Similar case. In mail from 2015-11-12 (path v2) you removed v9.4 compatibility,
so I'm not sure whether checking for 9.4 or 9.5 makes any sense now.
This review focuses mostly on documentation, but I went through both
documentation and code. I understood most of the code (and it makes
sense after some cosideration :-) ), but I'm not proficient in PostgreSQL
to be fully sure that there are no hidden bugs.
At the same time - I haven't seen problems and suspicious fragments of code,
so after fixing mentioned problems it should go to the commiter.
Best regards.
The new status of this patch is: Waiting on Author
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Sun, Jan 3, 2016 at 7:21 PM, Tomasz Rybak <tomasz.rybak@post.pl> wrote:
The following review has been posted through the commitfest application:
make installcheck-world: tested, passed
Implements feature: not tested
Spec compliant: not tested
Documentation: tested, failedApplies cleanly on current master
(b416c0bb622ce5d33fdbec3bbce00451132f10ec).Builds without any problems on current Debian unstable (am64 arch, GCC
5.3.1-4, glibc 2.21-6)
A make from an external build dir fails on install, suggested fix:
install: all
$(MKDIR_P) '$(DESTDIR)$(includedir)'/pglogical_output
- $(INSTALL_DATA) pglogical_output/hooks.h
'$(DESTDIR)$(includedir)'/pglogical_output
+ $(INSTALL_DATA)
$(top_srcdir)/contrib/pglogical_output/pglogical_output/hooks.h
'$(DESTDIR)$(includedir)'/pglogical_output
--
Alex
Shulgin, Oleksandr wrote:
A make from an external build dir fails on install, suggested fix:
install: all $(MKDIR_P) '$(DESTDIR)$(includedir)'/pglogical_output - $(INSTALL_DATA) pglogical_output/hooks.h '$(DESTDIR)$(includedir)'/pglogical_output + $(INSTALL_DATA) $(top_srcdir)/contrib/pglogical_output/pglogical_output/hooks.h '$(DESTDIR)$(includedir)'/pglogical_output
Actually you should be able to use $(srcdir)/hooks.h ...
--
�lvaro Herrera http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 12/22/15 4:55 AM, Craig Ringer wrote:
I'm a touch frustrated by that, as a large part of the point of
submitting the output plugin separately and in advance of the downstream
was to get attention for it separately, as its own entity. A lot of
effort has been put into making this usable for more than just a data
source for pglogical's replication tools.
I can't imagine that there is a lot of interest in a replication tool
where you only get one side of it, no matter how well-designed or
general it is. Ultimately, what people will want to do with this is
replicate things, not muse about its design aspects. So if we're going
to ship a replication solution in PostgreSQL core, we should ship all
the pieces that make the whole system work.
Also, I think there are two kinds of general systems: common core, and
all possible features. A common core approach could probably be made
acceptable with the argument that anyone will probably want to do things
this way, so we might as well implement it once and give it to people.
In a way, the logical decoding interface is the common core, as we
currently understand it. But this submission clearly has a lot of
features beyond just the basics, and we could probably go through them
one by one and ask, why do we need this bit? So that kind of system
will be very hard to review as a standalone submission.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Wed, Jan 6, 2016 at 5:17 PM, Peter Eisentraut <peter_e@gmx.net> wrote:
I can't imagine that there is a lot of interest in a replication tool
where you only get one side of it, no matter how well-designed or
general it is.
Well I do have another purpose in mind myself so I do appreciate it
being available now and separately.
However you also need to keep in mind that any of these other purposes
will be more or less equally large projects as logical replication.
There's no particular reason to expect one to be able to start up
today and provide feedback faster than the replication code that's
already been under development for ages. I haven't even started on my
pet project and probably won't until February. And I haven't even
thought through the details of it so I don't even know if it'll be a
matter of a few weeks or months or more.
The one project that does seem like it should be fairly fast to get
going and provide a relatively easy way to test the APIs separately
would be an auditing tool. I saw one go by but didn't look into
whether it used logical decoding or another mechanism. One based on
logical decoding does seem like it would let you verify that, for
example, the api gave the right information to filter effectively and
store meta information to index the audit records effectively.
--
greg
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 7 January 2016 at 01:17, Peter Eisentraut <peter_e@gmx.net> wrote:
On 12/22/15 4:55 AM, Craig Ringer wrote:
I'm a touch frustrated by that, as a large part of the point of
submitting the output plugin separately and in advance of the downstream
was to get attention for it separately, as its own entity. A lot of
effort has been put into making this usable for more than just a data
source for pglogical's replication tools.I can't imagine that there is a lot of interest in a replication tool
where you only get one side of it, no matter how well-designed or
general it is.
Well, the other part was posted most of a week ago.
/messages/by-id/5685BB86.5010901@2ndquadrant.com
... but this isn't just about replication. At least, not just to another
PostgreSQL instance. This plugin is designed to be general enough to use
for replication to other DBMSes (via appropriate receivers), to replace
trigger-based data collection in existing replication systems, for use in
audit data collection, etc.
Want to get a stream of data out of PostgreSQL in a consistent, simple way,
without having to add triggers or otherwise interfere with the origin
database? That's the purpose of this plugin, and it doesn't care in the
slightest what the receiver wants to do with that data. It's been designed
to be usable separately from pglogical downstream and - before the Python
tests were rejected in discussions on this list - was tested using a test
suite completely separate to the pglogical downstream using psycopg2 to
make sure no unintended interdependencies got introduced.
You can do way more than that with the output plugin but you have to write
your own downstream/receiver for the desired purpose, since using a
downstream based on bgworkers and SPI won't make any sense outside
PostgreSQL.
If you just want a canned product to use, see the pglogical post above for
the downstream code.
Ultimately, what people will want to do with this is
replicate things, not muse about its design aspects. So if we're going
to ship a replication solution in PostgreSQL core, we should ship all
the pieces that make the whole system work.
I don't buy that argument. Doesn't that mean logical decoding shouldn't
have been accepted? Or the initial patches for parallel query? Or any
number of other things that're part of incremental development solutions?
(This also seems to contradict what you then argue below, that the proposed
feature is too broad and does too much.)
I'd be happy to see both parts go in, but I'm frustrated that nobody's
willing to see beyond "replicate from one Pg to another Pg" and see all the
other things you can do. Want to replicate to Oracle / MS-SQL / etc? This
will help a lot and solve a significant part of the problem for you. Want
to stream data to append-only audit logs? Ditto. But nope, it's all about
PostgreSQL to PostgreSQL.
Please try to look further into what client applications can do with this
directly. I already know it meets the needs of the pglogical downstream.
What I was hoping to achieve with posting the output plugin earlier was to
get some thought going about what *else* it'd be good for.
Again: pglogical is posted now (it just took longer than expected to get
ready) and I'll be happy to see both it and the output plugin included. I
just urge people to look at the output plugin as more than a tightly
coupled component of pglogical.
Maybe some quality name bikeshedding for the output plugin would help ;)
Also, I think there are two kinds of general systems: common core, and
all possible features. A common core approach could probably be made
acceptable with the argument that anyone will probably want to do things
this way, so we might as well implement it once and give it to people.
That's what we're going for here. Extensible, something people can build on
and use.
In a way, the logical decoding interface is the common core, as we
currently understand it. But this submission clearly has a lot of
features beyond just the basics
Really? What would you cut? What's beyond the basics here? What basics are
you thinking of, i.e what set of requirements are you working towards /
needs are you seeking to meet?
We cut this to the bone to produce a minimum viable logical replication
solution. Especially the output plugin.
Cut the hook interfaces for row and xact filtering? You lose the ability to
use replication origins, crippling functionality, and for no real gain in
simplicity.
Remove JSON support? That's what most people are actually likely to want to
use when using the output plugin directly, and it's important for
debugging/tracing/diagnostics. It's a separate feature, to be sure, but
it's also a pretty trivial addition.
and we could probably go through them
one by one and ask, why do we need this bit? So that kind of system
will be very hard to review as a standalone submission.
Again, I disagree. I think you're looking at this way too narrowly.
I find it quite funny, actually. Here we go and produce something that's a
nice re-usable component that other people can use in their products and
solutions ... and all anyone does is complain that the other part required
to use it as a canned product isn't posted yet (though it is now). But with
BDR all anyone ever does is complain that it's too tightly coupled to the
needs of a single product and the features extracted from it, like
replication origins, should be more generic and general purpose so other
people can use them in their products too. Which is it going to be?
It would be helpful if you could take a step back and describe what *you*
think logical replication for PostgreSQL should look like. You clearly have
a picture in mind of what it should be, what requirements it satisfies,
etc. If you're going to argue based on that it'd be very helpful to
describe it. I might've missed some important points you've seen and you
might've overlooked issues I've seen.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
On 7 January 2016 at 02:32, Greg Stark <stark@mit.edu> wrote:
However you also need to keep in mind that any of these other purposes
will be more or less equally large projects as logical replication.
There's no particular reason to expect one to be able to start up
today and provide feedback faster than the replication code that's
already been under development for ages.
Good point. Though it's been out there for a while now.
The one project that does seem like it should be fairly fast to get
going and provide a relatively easy way to test the APIs separately
would be an auditing tool.
Yep. The json stream is very simple to consume for that purpose too,
providing pre-jsonified data from the upstream so you don't have to deal
with all the mess of maintaining matching type and table definitions on the
downstream to construct HeapTuple s, etc.
You generally don't want to write audit logs into a relation of the same
form as the original table plus some extra columns. It's nicer to work
with, sure, but it means you're almost totally locked in to your original
table definition. You can add new nullable columns but that's about it,
since there's no way to retroactively construct audit data for already
logged entries. So I think the json output mode is more interesting for
auditing.
An interesting hiccup for audit is that AFAIK nothing in WAL associates
user identity information with a change, so we can't decode and send info
on which user performed which xact or changed a given tuple. SET ROLE and
SECURITY DEFINER mean there isn't a 1:1 mapping of xact to user either, and
info would probably have to be tuple level. Fun. The WAL messages patch
would be helpful for this sort of thing, allowing apps to use triggers to
add arbitrary info to xlog; XLOG_NOOP could help too, just in an uglier way
with more room for confusion between unrelated products using it.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
Here's v5 of the pglogical patch.
Changes:
* Implement relation metadata caching
* Add the relmeta_cache_size parameter for cache control
* Add an extension to get version information
* Create the pglogical_output header directory on install
* Restore 9.4 compatibility (it's small)
* Allow row filter hooks to see details of the changed tuple
* Remove forward_changesets from pglogical_output (use a hook if you want
this functionality)
I'm not sure if 9.4 compat will be desirable or not. It's handy to avoid
needing a separate backported version, but also confusing to do a PGXS
build within a 9.6 tree against 9.4...
Attachments:
pglogical-output-v5.patchtext/x-patch; charset=UTF-8; name=pglogical-output-v5.patchDownload
From 6941b204141c3ab943fa377993d9ea8602993b31 Mon Sep 17 00:00:00 2001
From: Craig Ringer <craig@2ndquadrant.com>
Date: Mon, 2 Nov 2015 19:34:21 +0800
Subject: [PATCH] Add contrib/pglogical_output, a logical decoding plugin
pglogical v5
---
contrib/Makefile | 2 +
contrib/pglogical_output/.gitignore | 7 +
contrib/pglogical_output/Makefile | 71 +++
contrib/pglogical_output/README.md | 612 +++++++++++++++++++++
contrib/pglogical_output/doc/.gitignore | 1 +
contrib/pglogical_output/doc/DESIGN.md | 230 ++++++++
contrib/pglogical_output/doc/protocol.txt | 544 ++++++++++++++++++
contrib/pglogical_output/expected/basic_json.out | 140 +++++
contrib/pglogical_output/expected/basic_json_1.out | 139 +++++
contrib/pglogical_output/expected/basic_native.out | 113 ++++
contrib/pglogical_output/expected/cleanup.out | 4 +
.../pglogical_output/expected/encoding_json.out | 59 ++
contrib/pglogical_output/expected/extension.out | 14 +
contrib/pglogical_output/expected/hooks_json.out | 204 +++++++
contrib/pglogical_output/expected/hooks_json_1.out | 202 +++++++
contrib/pglogical_output/expected/hooks_native.out | 104 ++++
.../pglogical_output/expected/params_native.out | 133 +++++
.../pglogical_output/expected/params_native_1.out | 134 +++++
contrib/pglogical_output/expected/prep.out | 26 +
contrib/pglogical_output/pglogical_config.c | 526 ++++++++++++++++++
contrib/pglogical_output/pglogical_config.h | 55 ++
contrib/pglogical_output/pglogical_hooks.c | 235 ++++++++
contrib/pglogical_output/pglogical_hooks.h | 23 +
contrib/pglogical_output/pglogical_infofuncs.c | 55 ++
.../pglogical_output/pglogical_output--1.0.0.sql | 11 +
contrib/pglogical_output/pglogical_output.c | 569 +++++++++++++++++++
.../pglogical_output/pglogical_output.control.in | 4 +
contrib/pglogical_output/pglogical_output.h | 111 ++++
contrib/pglogical_output/pglogical_output/README | 7 +
contrib/pglogical_output/pglogical_output/compat.h | 28 +
contrib/pglogical_output/pglogical_output/hooks.h | 73 +++
contrib/pglogical_output/pglogical_proto.c | 49 ++
contrib/pglogical_output/pglogical_proto.h | 61 ++
contrib/pglogical_output/pglogical_proto_json.c | 204 +++++++
contrib/pglogical_output/pglogical_proto_json.h | 32 ++
contrib/pglogical_output/pglogical_proto_native.c | 513 +++++++++++++++++
contrib/pglogical_output/pglogical_proto_native.h | 38 ++
contrib/pglogical_output/pglogical_relmetacache.c | 194 +++++++
contrib/pglogical_output/pglogical_relmetacache.h | 19 +
contrib/pglogical_output/regression.conf | 2 +
contrib/pglogical_output/sql/basic_json.sql | 24 +
contrib/pglogical_output/sql/basic_native.sql | 37 ++
contrib/pglogical_output/sql/basic_setup.sql | 62 +++
contrib/pglogical_output/sql/basic_teardown.sql | 4 +
contrib/pglogical_output/sql/cleanup.sql | 4 +
contrib/pglogical_output/sql/encoding_json.sql | 58 ++
contrib/pglogical_output/sql/extension.sql | 7 +
contrib/pglogical_output/sql/hooks_json.sql | 49 ++
contrib/pglogical_output/sql/hooks_native.sql | 48 ++
contrib/pglogical_output/sql/hooks_setup.sql | 37 ++
contrib/pglogical_output/sql/hooks_teardown.sql | 10 +
contrib/pglogical_output/sql/params_native.sql | 104 ++++
contrib/pglogical_output/sql/prep.sql | 30 +
contrib/pglogical_output_plhooks/.gitignore | 1 +
contrib/pglogical_output_plhooks/Makefile | 13 +
.../README.pglogical_output_plhooks | 158 ++++++
.../pglogical_output_plhooks--1.0.sql | 89 +++
.../pglogical_output_plhooks.c | 414 ++++++++++++++
.../pglogical_output_plhooks.control | 4 +
59 files changed, 6701 insertions(+)
create mode 100644 contrib/pglogical_output/.gitignore
create mode 100644 contrib/pglogical_output/Makefile
create mode 100644 contrib/pglogical_output/README.md
create mode 100644 contrib/pglogical_output/doc/.gitignore
create mode 100644 contrib/pglogical_output/doc/DESIGN.md
create mode 100644 contrib/pglogical_output/doc/protocol.txt
create mode 100644 contrib/pglogical_output/expected/basic_json.out
create mode 100644 contrib/pglogical_output/expected/basic_json_1.out
create mode 100644 contrib/pglogical_output/expected/basic_native.out
create mode 100644 contrib/pglogical_output/expected/cleanup.out
create mode 100644 contrib/pglogical_output/expected/encoding_json.out
create mode 100644 contrib/pglogical_output/expected/extension.out
create mode 100644 contrib/pglogical_output/expected/hooks_json.out
create mode 100644 contrib/pglogical_output/expected/hooks_json_1.out
create mode 100644 contrib/pglogical_output/expected/hooks_native.out
create mode 100644 contrib/pglogical_output/expected/params_native.out
create mode 100644 contrib/pglogical_output/expected/params_native_1.out
create mode 100644 contrib/pglogical_output/expected/prep.out
create mode 100644 contrib/pglogical_output/pglogical_config.c
create mode 100644 contrib/pglogical_output/pglogical_config.h
create mode 100644 contrib/pglogical_output/pglogical_hooks.c
create mode 100644 contrib/pglogical_output/pglogical_hooks.h
create mode 100644 contrib/pglogical_output/pglogical_infofuncs.c
create mode 100644 contrib/pglogical_output/pglogical_output--1.0.0.sql
create mode 100644 contrib/pglogical_output/pglogical_output.c
create mode 100644 contrib/pglogical_output/pglogical_output.control.in
create mode 100644 contrib/pglogical_output/pglogical_output.h
create mode 100644 contrib/pglogical_output/pglogical_output/README
create mode 100644 contrib/pglogical_output/pglogical_output/compat.h
create mode 100644 contrib/pglogical_output/pglogical_output/hooks.h
create mode 100644 contrib/pglogical_output/pglogical_proto.c
create mode 100644 contrib/pglogical_output/pglogical_proto.h
create mode 100644 contrib/pglogical_output/pglogical_proto_json.c
create mode 100644 contrib/pglogical_output/pglogical_proto_json.h
create mode 100644 contrib/pglogical_output/pglogical_proto_native.c
create mode 100644 contrib/pglogical_output/pglogical_proto_native.h
create mode 100644 contrib/pglogical_output/pglogical_relmetacache.c
create mode 100644 contrib/pglogical_output/pglogical_relmetacache.h
create mode 100644 contrib/pglogical_output/regression.conf
create mode 100644 contrib/pglogical_output/sql/basic_json.sql
create mode 100644 contrib/pglogical_output/sql/basic_native.sql
create mode 100644 contrib/pglogical_output/sql/basic_setup.sql
create mode 100644 contrib/pglogical_output/sql/basic_teardown.sql
create mode 100644 contrib/pglogical_output/sql/cleanup.sql
create mode 100644 contrib/pglogical_output/sql/encoding_json.sql
create mode 100644 contrib/pglogical_output/sql/extension.sql
create mode 100644 contrib/pglogical_output/sql/hooks_json.sql
create mode 100644 contrib/pglogical_output/sql/hooks_native.sql
create mode 100644 contrib/pglogical_output/sql/hooks_setup.sql
create mode 100644 contrib/pglogical_output/sql/hooks_teardown.sql
create mode 100644 contrib/pglogical_output/sql/params_native.sql
create mode 100644 contrib/pglogical_output/sql/prep.sql
create mode 100644 contrib/pglogical_output_plhooks/.gitignore
create mode 100644 contrib/pglogical_output_plhooks/Makefile
create mode 100644 contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
create mode 100644 contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
diff --git a/contrib/Makefile b/contrib/Makefile
index bd251f6..028fd9a 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -35,6 +35,8 @@ SUBDIRS = \
pg_stat_statements \
pg_trgm \
pgcrypto \
+ pglogical_output \
+ pglogical_output_plhooks \
pgrowlocks \
pgstattuple \
postgres_fdw \
diff --git a/contrib/pglogical_output/.gitignore b/contrib/pglogical_output/.gitignore
new file mode 100644
index 0000000..6cdc3f1
--- /dev/null
+++ b/contrib/pglogical_output/.gitignore
@@ -0,0 +1,7 @@
+pglogical_output.so
+results/
+regression.diffs
+tmp_install/
+tmp_check/
+log/
+pglogical_output.control
diff --git a/contrib/pglogical_output/Makefile b/contrib/pglogical_output/Makefile
new file mode 100644
index 0000000..e63aa21
--- /dev/null
+++ b/contrib/pglogical_output/Makefile
@@ -0,0 +1,71 @@
+MODULE_big = pglogical_output
+PGFILEDESC = "pglogical_output - logical replication output plugin"
+
+OBJS = pglogical_output.o pglogical_hooks.o pglogical_config.o \
+ pglogical_proto.o pglogical_proto_native.o \
+ pglogical_proto_json.o pglogical_relmetacache.o \
+ pglogical_infofuncs.o
+
+REGRESS = prep params_native basic_native hooks_native basic_json hooks_json encoding_json extension cleanup
+
+EXTENSION = pglogical_output
+DATA = pglogical_output--1.0.0.sql
+EXTRA_CLEAN += pglogical_output.control
+
+
+ifdef USE_PGXS
+
+# For regression checks
+# http://www.postgresql.org/message-id/CAB7nPqTsR5o3g-fBi6jbsVdhfPiLFWQ_0cGU5=94Rv_8W3qvFA@mail.gmail.com
+# this makes "make check" give a useful error
+abs_top_builddir = .
+NO_TEMP_INSTALL = yes
+# Usual recipe
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+
+# These don't do anything yet, since temp install is disabled
+EXTRA_INSTALL += ./examples/hooks
+REGRESS_OPTS += --temp-config=regression.conf
+
+plhooks:
+ make -C examples/hooks USE_PGXS=1 clean install
+
+installcheck: plhooks
+
+else
+
+subdir = contrib/pglogical_output
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# 'make installcheck' disabled when building in-tree because these tests
+# require "wal_level=logical", which typical installcheck users do not have
+# (e.g. buildfarm clients).
+installcheck:
+ ;
+
+EXTRA_INSTALL += contrib/pglogical_output_plhooks
+EXTRA_REGRESS_OPTS += --temp-config=./regression.conf
+
+install: all
+
+endif
+
+# The # in #define is taken as a comment, per https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=142043
+# so it must be escaped. The $ placeholders in awk must be doubled too.
+pglogical_output_version=$(shell awk '/\#define PGLOGICAL_OUTPUT_VERSION[ \t]+\".*\"/ { print substr($$3,2,length($$3)-2) }' pglogical_output.h )
+
+all: pglogical_output.control
+
+pglogical_output.control: pglogical_output.control.in pglogical_output.h
+ sed 's/__PGLOGICAL_OUTPUT_VERSION__/$(pglogical_output_version)/' pglogical_output.control.in > pglogical_output.control
+
+install: header_install
+
+header_install: pglogical_output/compat.h pglogical_output/hooks.h
+ $(MKDIR_P) '$(DESTDIR)$(includedir)'/pglogical_output
+ $(INSTALL_DATA) pglogical_output/compat.h '$(DESTDIR)$(includedir)'/pglogical_output
+ $(INSTALL_DATA) pglogical_output/hooks.h '$(DESTDIR)$(includedir)'/pglogical_output
diff --git a/contrib/pglogical_output/README.md b/contrib/pglogical_output/README.md
new file mode 100644
index 0000000..548f1d5
--- /dev/null
+++ b/contrib/pglogical_output/README.md
@@ -0,0 +1,612 @@
+# `pglogical` Output Plugin
+
+This is the [logical decoding](http://www.postgresql.org/docs/current/static/logicaldecoding.html)
+[output plugin](http://www.postgresql.org/docs/current/static/logicaldecoding-output-plugin.html)
+for `pglogical`. Its purpose is to extract a change stream from a PostgreSQL
+database and send it to a client over a network connection using a
+well-defined, efficient protocol that multiple different applications can
+consume.
+
+The primary purpose of `pglogical_output` is to supply data to logical
+streaming replication solutions, but any application can potentially use its
+data stream. The output stream is designed to be compact and fast to decode,
+and the plugin supports upstream filtering of data so that only the required
+information is sent.
+
+Only one database is replicated, rather than the whole PostgreSQL install. A
+subset of that database may be selected for replication, currently based on
+table and on replication origin. Filtering by a WHERE clause can be supported
+easily in future.
+
+No triggers are required to collect the change stream and no external ticker or
+other daemon is required. It's accumulated using
+[replication slots](http://www.postgresql.org/docs/current/static/logicaldecoding-explanation.html#AEN66446),
+as supported in PostgreSQL 9.4 or newer, and sent on top of the
+[PostgreSQL streaming replication protocol](http://www.postgresql.org/docs/current/static/protocol-replication.html).
+
+Unlike block-level ("physical") streaming replication, the change stream from
+the `pglogical` output plugin is compatible across different PostgreSQL
+versions and can even be consumed by non-PostgreSQL clients.
+
+Because logical decoding is used, only the changed rows are sent on the wire.
+There's no index change data, no vacuum activity, etc transmitted.
+
+The use of a replication slot means that the change stream is reliable and
+crash-safe. If the client disconnects or crashes it can reconnect and resume
+replay from the last message that client processed. Server-side changes that
+occur while the client is disconnected are accumulated in the queue to be sent
+when the client reconnects. This reliability also means that server-side
+resources are consumed whether or not a client is connected.
+
+# Why another output plugin?
+
+See [`DESIGN.md`](DESIGN.md) for a discussion of why using one of the existing
+generic logical decoding output plugins like `wal2json` to drive a logical
+replication downstream isn't ideal. It's mostly about speed.
+
+# Architecture and high level interaction
+
+The output plugin is loaded by a PostgreSQL walsender process when a client
+connects to PostgreSQL using the PostgreSQL wire protocol with connection
+option `replication=database`, then uses
+[the `CREATE_REPLICATION_SLOT ... LOGICAL ...` or `START_REPLICATION SLOT ... LOGICAL ...` commands](http://www.postgresql.org/docs/current/static/logicaldecoding-walsender.html) to start streaming changes. (It can also be used via
+[SQL level functions](http://www.postgresql.org/docs/current/static/logicaldecoding-sql.html)
+over a non-replication connection, but this is mainly for debugging purposes).
+
+The client supplies parameters to the `START_REPLICATION SLOT ... LOGICAL ...`
+command to specify the version of the `pglogical` protocol it supports,
+whether it wants binary format, etc.
+
+The output plugin processes the connection parameters and the connection enters
+streaming replication protocol mode, sometimes called "COPY BOTH" mode because
+it's based on the protocol used for the `COPY` command. PostgreSQL then calls
+functions in this plugin to send it a stream of transactions to decode and
+translate into network messages. This stream of changes continues until the
+client disconnects.
+
+The only client-to-server interaction after startup is the sending of periodic
+feedback messages that allow the replication slot to discard no-longer-needed
+change history. The client *must* send feedback, otherwise `pg_xlog` on the
+server will eventually fill up and the server will stop working.
+
+
+# Usage
+
+The overall flow of client/server interaction is:
+
+* Client makes PostgreSQL fe/be protocol connection to server
+ * Connection options must include `replication=database` and `dbname=[...]` parameters
+ * The PostgreSQL client library can be `libpq` or anything else that supports the replication sub-protocol
+ * The same mechanisms are used for authentication and protocol encryption as for a normal non-replication connection
+* Client issues `IDENTIFY_SYSTEM`
+ * Server responds with a single row containing system identity info
+* Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'` if it's setting up for the first time
+ * Server responds with success info and a snapshot identifier
+ * Client may at this point use the snapshot identifier on other connections while leaving this one idle
+* Client issues `START_REPLICATION SLOT slotname LOGICAL 0/0 (...options...)` to start streaming, which loops:
+ * Server emits `pglogical` message block encapsulated in a replication protocol `CopyData` message
+ * Client receives and unwraps message, then decodes the `pglogical` message block
+ * Client intermittently sends a standby status update message to server to confirm replay
+* ... until client sends a graceful connection termination message on the fe/be protocol level or the connection is broken
+
+ The details of `IDENTIFY_SYSTEM`, `CREATE_REPLICATION_SLOT` and `START_REPLICATION` are discussed in the [replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html) and will not be repeated here.
+
+## Make a replication connection
+
+To use the `pglogical` plugin you must first establish a PostgreSQL FE/BE
+protocol connection using the client library of your choice, passing
+`replication=database` as one of the connection parameters. `database` is a
+literal string and is not replaced with the database name; instead the database
+name is passed separately in the usual `dbname` parameter. Note that
+`replication` is not a GUC (configuration parameter) and may not be passed in
+the `options` parameter on the connection, it's a top-level parameter like
+`user` or `dbname`.
+
+Example connection string for `libpq`:
+
+ 'user=postgres replication=database sslmode=verify-full dbname=mydb'
+
+The plug-in name to pass on logical slot creation is `'pglogical'`.
+
+Details are in the replication protocol docs.
+
+## Get system identity
+
+If required you can use the `IDENTIFY_SYSTEM` command, which reports system
+information:
+
+ systemid | timeline | xlogpos | dbname | dboid
+ ---------------------+----------+-----------+--------+-------
+ 6153224364663410513 | 1 | 0/C429C48 | testd | 16385
+ (1 row)
+
+Details are in the replication protocol docs.
+
+## Create the slot if required
+
+If your application creates its own slots on first use and hasn't previously
+connected to this database on this system you'll need to create a replication
+slot. This keeps track of the client's replay state even while it's disconnected.
+
+The slot name may be anything your application wants up to a limit of 63
+characters in length. It's strongly advised that the slot name clearly identify
+the application and the host it runs on.
+
+Pass `pglogical` as the plugin name.
+
+e.g.
+
+ CREATE_REPLICATION_SLOT "reporting_host_42" LOGICAL "pglogical";
+
+`CREATE_REPLICATION_SLOT` returns a snapshot identifier that may be used with
+[`SET TRANSACTION SNAPSHOT`](http://www.postgresql.org/docs/current/static/sql-set-transaction.html)
+to see the database's state as of the moment of the slot's creation. The first
+change streamed from the slot will be the change immediately after this
+snapshot was taken. The snapshot is useful when cloning the initial state of a
+database being replicted. Applications that want to see the change stream
+going forward, but don't care about the initial state, can ignore this. The
+snapshot is only valid as long as the connection that issued the
+`CREATE_REPLICATION_SLOT` remains open and has not run another command.
+
+## Send replication parameters
+
+The client now sends:
+
+ START_REPLICATION SLOT "the_slot_name" LOGICAL (
+ 'Expected_encoding', 'UTF8',
+ 'Max_proto_major_version', '1',
+ 'Min_proto_major_version', '1',
+ ...moreparams...
+ );
+
+to start replication.
+
+The parameters are very important for ensuring that the plugin accepts
+the replication request and streams changes in the expected form. `pglogical`
+parameters are discussed in the separate `pglogical` protocol documentation.
+
+## Process the startup message
+
+`pglogical`'s output plugin will send a `CopyData` message containing its
+startup message as the first protocol message. This message contains a
+set of key/value entries describing the capabilities of the upstream output
+plugin, its version and the Pg version, the tuple format options selected,
+etc.
+
+The downstream client may choose to cleanly close the connection and disconnect
+at this point if it doesn't like the reply. It might then inform the user
+or reconnect with different parameters based on what it learned from the
+first connection's startup message.
+
+## Consume the change stream
+
+`pglogical`'s output plugin now sends a continuous series of `CopyData`
+protocol messages, each of which encapsulates a `pglogical` protocol message
+as documented in the separate protocol docs.
+
+These messages provide information about transaction boundaries, changed
+rows, etc.
+
+The stream continues until the client disconnects, the upstream server is
+restarted, the upstream walsender is terminated by admin action, there's
+a network issue, or the connection is otherwise broken.
+
+The client should send periodic feedback messages to the server to acknowledge
+that it's replayed to a given point and let the server release the resources
+it's holding in case that change stream has to be replayed again. See
+["Hot standby feedback message" in the replication protocol docs](http://www.postgresql.org/docs/current/static/protocol-replication.html)
+for details.
+
+## Disconnect gracefully
+
+Disconnection works just like any normal client; you use your client library's
+usual method for closing the connection. No special action is required before
+disconnection, though it's usually a good idea to send a final standby status
+message just before you disconnect.
+
+# Tests
+
+The `pg_regress` tests check invalid parameter handling and basic
+functionality. They're intended for use by the buildfarm using an in-tree
+`make check`, but may also be run with an out-of-tree PGXS build against an
+existing PostgreSQL install using `make USE_PGXS=1 clean installcheck`.
+
+The tests may fail on installations that are not utf-8 encoded because the
+payloads of the binary protocol output will have text in different encodings,
+which aren't visible to psql as text to be decoded. Avoiding anything except
+7-bit ascii in the tests *should* prevent the problem.
+
+# Changeset forwarding
+
+It's possible to use `pglogical_output` to cascade replication between multiple
+PostgreSQL servers, in combination with an appropriate client to apply the
+changes to the downstreams.
+
+There are two forwarding modes:
+
+* Forward everything. Transactions are replicated whether they were made directly
+ on the immediate upstream or some other node upstream of it. All rows from all
+ transactions are sent.
+
+ Selected by not setting a row or transaction filter hook.
+
+* Filtered forwarding. Transactions are replicated unless a client-supplied
+ transaction filter hook says to skip this transaction. Row changes are
+ replicated unless the client-supplied row filter hook (if provided) says to
+ skip that row.
+
+ Selected by installing a transaction and/or row filter hook (see "hooks").
+
+If the upstream server is 9.5 or newer the server will enable changeset origin
+information. It will set `forward_changeset_origins` to true in the startup
+reply message to indicate this. It will then send changeset origin messages
+after the `BEGIN` for each transaction, per the protocol documentation. Origin
+messages are omitted for transactions originating directly on the immediate
+upstream to save bandwidth. If `forward_changeset_origins` is true then
+transactions without an origin are always from the immediate upstream that’s
+running the decoding plugin.
+
+Note that 9.4 servers can't expose replication origin information so they pass
+zero to the row filter hook and don't call the transaction filter hook.
+
+Clients may use this facility to form arbitrarily complex topologies when
+combined with hooks to determine which transactions are forwarded. An obvious
+case is bi-directional (mutual) replication.
+
+# Selective replication
+
+By specifying a row filter hook it's possible to filter the replication stream
+server-side so that only a subset of changes is replicated.
+
+
+# Hooks
+
+`pglogical_output` exposes a number of extension points where applications can
+modify or override its behaviour.
+
+All hooks are called in their own memory context, which lasts for the duration
+of the logical decoding session. They may switch to longer lived contexts if
+needed, but are then responsible for their own cleanup.
+
+## Hook setup function
+
+The downstream must specify the fully-qualified name of a SQL-callable function
+on the server as the value of the `hooks.setup_function` client parameter.
+The SQL signature of this function is
+
+ CREATE OR REPLACE FUNCTION funcname(hooks internal, memory_context internal)
+ RETURNS void STABLE
+ LANGUAGE c AS 'MODULE_PATHNAME';
+
+Permissions are checked. This function must be callable by the user that the
+output plugin is running as. The function name *must* be schema-qualified and is
+parsed like any other qualified identifier.
+
+The function receives a pointer to a newly allocated structure of hook function
+pointers to populate as its first argument. The function must not free the
+argument.
+
+If the hooks need a private data area to store information across calls, the
+setup function should get the `MemoryContext` pointer from the 2nd argument,
+then `MemoryContextAlloc` a struct for the data in that memory context and
+store the pointer to it in `hooks->hooks_private_data`. This will then be
+accessible on future calls to hook functions. It need not be manually freed, as
+the memory context used for logical decoding will free it when it's freed.
+Don't put anything in it that needs manual cleanup.
+
+Each hook has its own C signature (defined below) and the pointers must be
+directly to the functions. Hooks that the client does not wish to set must be
+left null.
+
+An example is provided in `contrib/pglogical_output_plhooks` and the argument
+structs are defined in `pglogical_output/hooks.h`, which is installed into the
+PostgreSQL source tree when the extension is installed.
+
+Each hook that is enabled results in a new startup parameter being emitted in
+the startup reply message. Clients must check for these and must not assume a
+hook was successfully activated because no error is seen.
+
+Hook functions are called in the context of the backend doing logical decoding.
+Except for the startup hook, hooks see the catalog state as it was at the time
+the transaction or row change being examined was made. Access to to non-catalog
+tables is unsafe unless they have the `user_catalog_table` reloption set.
+
+## Startup hook
+
+The startup hook is called when logical decoding starts.
+
+This hook can inspect the parameters passed by the client to the output
+plugin as in_params. These parameters *must not* be modified.
+
+It can add new parameters to the set to be returned to the client in the
+startup parameters message, by appending to List out_params, which is
+initially NIL. Each element must be a `DefElem` with the param name
+as the `defname` and a `String` value as the arg, as created with
+`makeDefElem(...)`. It and its contents must be allocated in the
+logical decoding memory context.
+
+For walsender based decoding the startup hook is called only once, and
+cleanup might not be called at the end of the session.
+
+Multiple decoding sessions, and thus multiple startup hook calls, may happen
+in a session if the SQL interface for logical decoding is being used. In
+that case it's guaranteed that the cleanup hook will be called between each
+startup.
+
+When successfully enabled, the output parameter `hooks.startup_hook_enabled` is
+set to true in the startup reply message.
+
+Unlike the other hooks, this hook sees a snapshot of the database's current
+state, not a time-traveled catalog state. It is safe to access all tables from
+this hook.
+
+## Transaction filter hook
+
+The transaction filter hook can exclude entire transactions from being decoded
+and replicated based on the node they originated from.
+
+It is passed a `const TxFilterHookArgs *` containing:
+
+* The hook argument supplied by the client, if any
+* The `RepOriginId` that this transaction originated from
+
+and must return boolean, where true retains the transaction for sending to the
+client and false discards it. (Note that this is the reverse sense of the low
+level logical decoding transaction filter hook).
+
+The hook function must *not* free the argument struct or modify its contents.
+
+Note that individual changes within a transaction may have different origins to
+the transaction as a whole; see "Origin filtering" for more details. If a
+transaction is filtered out, all changes are filtered out even if their origins
+differ from that of the transaction as a whole.
+
+When successfully enabled, the output parameter
+`hooks.transaction_filter_enabled` is set to true in the startup reply message.
+
+## Row filter hook
+
+The row filter hook is called for each row. It is passed information about the
+table, the transaction origin, and the row origin.
+
+It is passed a `const RowFilterHookArgs*` containing:
+
+* The hook argument supplied by the client, if any
+* The `Relation` the change affects
+* The change type - 'I'nsert, 'U'pdate or 'D'elete
+
+It can return true to retain this row change, sending it to the client, or
+false to discard it.
+
+The function *must not* free the argument struct or modify its contents.
+
+Note that it is more efficient to exclude whole transactions with the
+transaction filter hook rather than filtering out individual rows.
+
+When successfully enabled, the output parameter
+`hooks.row_filter_enabled` is set to true in the startup reply message.
+
+## Shutdown hook
+
+The shutdown hook is called when a decoding session ends. You can't rely on
+this hook being invoked reliably, since a replication-protocol walsender-based
+session might just terminate. It's mostly useful for cleanup to handle repeated
+invocations under the SQL interface to logical decoding.
+
+You don't need a hook to free memory you allocated, unless you explicitly
+switched to a longer lived memory context like TopMemoryContext. Memory allocated
+in the hook context will be automatically when the decoding session shuts down.
+
+## Writing hooks in procedural languages
+
+You can write hooks in PL/PgSQL, etc, too, via the `pglogical_output_plhooks`
+adapter extension in `contrib`. They won't perform very well though.
+
+# Limitations
+
+The advantages of logical decoding in general and `pglogical_output` in
+particular are discussed above. There are also some limitations that apply to
+`pglogical_output`, and to Pg's logical decoding in general.
+
+(TODO: move much of this to the main logical decoding docs)
+
+Notably:
+
+## Doesn't replicate DDL
+
+Logical decoding doesn't decode catalog changes directly. So the plugin can't
+just send a `CREATE TABLE` statement when a new table is added.
+
+If the data being decoded is being applied to another PostgreSQL database then
+its table definitions must be kept in sync via some means external to the logical
+decoding plugin its self, such as:
+
+* Event triggers using DDL deparse to capture DDL changes as they happen and write them to a table to be replicated and applied on the other end; or
+* doing DDL management via tools that synchronise DDL on all nodes
+
+## Doesn't replicate global objects/shared catalog changes
+
+PostgreSQL has a number of object types that exist across all databases, stored
+in *shared catalogs*. These include:
+
+* Roles (users/groups)
+* Security labels on users and databases
+
+Such objects cannot be replicated by `pglogical_output`. They're managed with DDL that
+can't be captured within a single database and isn't decoded anyway.
+
+DDL for global object changes must be synchronized via some external means.
+
+## Mostly one-way communication
+
+Per the protocol documentation, the downstream can't send anything except
+replay progress messages to the upstream after replication begins, and can't
+re-initialise replication without a disconnect.
+
+To achieve downstream-to-upstream communication, clients can use a regular
+libpq connection to the upstream then write to tables or call functions.
+Alternately, a separate replication connection in the opposite direction can be
+created by the application to carry information from downstream to upstream.
+
+See "Protocol flow" in the protocol documentation for more information.
+
+## Physical replica failover
+
+Logical decoding cannot follow a physical replication failover because
+replication slot state is not replicated to physical replicas. If you fail over
+to a streaming replica you have to manually reconnect your logical replication
+clients, creating new slots, etc. This is a core PostgreSQL limitation.
+
+Also, there's no built-in way to guarantee that the logical replication slot
+from the failed master hasn't replayed further than the physical streaming
+replica you failed over to. You could receive changes on your logical decoding
+stream from the old master that never made it to the physical streaming
+replica. This is true (albeit very unlikely) *even if the physical streaming
+replica is synchronous* because PostgreSQL sends the replication data anyway,
+then just delays the commit's visibility on the master. Support for strictly
+ordered standbys would be required in PostgreSQL to avoid this.
+
+To achieve failover with logical replication you cannot mix in physical
+standbys. The logical replication client has to take responsibility for
+maintaining slots on logical replicas intended as failover candidates
+and for ensuring that the furthest-ahead replica is promoted if there is
+more than one.
+
+## Can only replicate complete transactions
+
+Logical decoding can only replicate a transaction after it has committed. This
+usefully skips replication of rolled back transactions, but it also means that
+very large transactions must be completed upstream before they can begin on the
+downstream, adding to replication latency.
+
+## Replicates only one transaction at a time
+
+Logical decoding serializes transactions in commit order, so pglogical_output
+cannot replay interleaved concurrent transactions. This can lead to high latencies
+when big transactions are being replayed, since smaller transactions get queued
+up behind them.
+
+## Unique index required for inserts or updates
+
+To replicate `INSERT`s or `UPDATE`s it is necessary to have a `PRIMARY KEY`
+or a (non-partial, columns-only) `UNIQUE` index on the table, so the table
+has a `REPLICA IDENTITY`. Without that `pglogical_output` doesn't know what
+old key to send to allow the receiver to tell which tuple is being updated.
+
+## UNLOGGED tables aren't replicated
+
+Because `UNLOGGED` tables aren't written to WAL, they aren't replicated by
+logical or physical replication. You can only replicate `UNLOGGED` tables
+with trigger-based solutions.
+
+## Unchanged fields are often sent in `UPDATE`
+
+Because there's no tracking of dirty/clean fields when a tuple is updated,
+logical decoding can't tell if a given field was changed by an update.
+Unchanged fields can only by identified and omitted if they're a variable
+length TOASTable type and are big enough to get stored out-of-line in
+a TOAST table.
+
+# Troubleshooting and debugging
+
+## Non-destructively previewing pending data on a slot
+
+Using the json mode of `pglogical_output` you can examine pending transactions
+on a slot without consuming them, so they are still delivered to the usual
+client application that created/owns this slot. This is best done using the SQL
+interface to logical decoding, since it gives you finer control than using
+`pg_recvlogical`.
+
+You can only peek at a slot while there is no other client connected to that
+slot.
+
+Use `pg_logical_slot_peek_changes` to examine the change stream without
+destructively consuming changes. This is extremely helpful when trying to
+determine why an error occurs in a downstream, since you can examine a
+json-ified representation of the xact. It's necessary to supply a minimal
+set of required parameters to the output plugin.
+
+e.g. given setup:
+
+ CREATE TABLE discard_test(blah text);
+ SELECT 'init' FROM pg_create_logical_replication_slot('demo_slot', 'pglogical_output');
+ INSERT INTO discard_test(blah) VALUES('one');
+ INSERT INTO discard_test(blah) VALUES('two1'),('two2'),('two3');
+ INSERT INTO discard_test(blah) VALUES('three1'),('three2');
+
+you can peek at the change stream with:
+
+ SELECT location, xid, data
+ FROM pg_logical_slot_peek_changes('demo_slot', NULL, NULL,
+ 'min_proto_version', '1', 'max_proto_version', '1',
+ 'startup_params_format', '1', 'proto_format', 'json');
+
+The two `NULL`s mean you don't want to stop decoding after any particular
+LSN or any particular number of changes. Decoding will stop when there's nothing
+left to decode or you cancel the query.
+
+This will emit a key/value startup message then change data rows like:
+
+ location | xid | data
+ 0/4E8AAF0 | 5562 | {"action":"B", has_catalog_changes:"f", xid:"5562", first_lsn:"0/4E8AAF0", commit_time:"2015-11-13 14:26:21.404425+08"}
+ 0/4E8AAF0 | 5562 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"one"}}
+ 0/4E8AB70 | 5562 | {"action":"C", final_lsn:"0/4E8AB30", end_lsn:"0/4E8AB70"}
+ 0/4E8ABA8 | 5563 | {"action":"B", has_catalog_changes:"f", xid:"5563", first_lsn:"0/4E8ABA8", commit_time:"2015-11-13 14:26:32.015611+08"}
+ 0/4E8ABA8 | 5563 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"two1"}}
+ 0/4E8ABE8 | 5563 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"two2"}}
+ 0/4E8AC28 | 5563 | {"action":"I","relation":["public","discard_test"],"newtuple":{"blah":"two3"}}
+ 0/4E8ACA8 | 5563 | {"action":"C", final_lsn:"0/4E8AC68", end_lsn:"0/4E8ACA8"}
+ ....
+
+The output is the LSN (log sequence number) associated with a change, the top
+level transaction ID that performed the change, and the change data as json.
+
+You can see the transaction boundaries by xid changes and by the "B"egin and
+"C"ommit messages, and you can see the individual row "I"nserts. Replication
+origins, commit timestamps, etc will be shown if known.
+
+See http://www.postgresql.org/docs/current/static/functions-admin.html for
+information on the peek functions.
+
+If you want the binary format you can get that with
+`pg_logical_slot_peek_binary_changes` and the `native` protocol, but that's
+generally much less useful.
+
+# Manually discarding a change from a slot
+
+Sometimes it's desirable to manually purge one or more changes from a
+replication slot. This is usually an error recovery step when problems arise
+with the downstream code that's replaying from the slot.
+
+You can use the peek functions to determine the point in the stream you want to
+discard up to, as identifed by LSN (log sequence number). See
+"non-destructively previewing pending data on a slot" above for details.
+
+You can't control the point you start discarding from, it's always from the
+current stream position up to a point you specify. If the peek shows that
+there's data you still want to retain you must make sure that the downstream
+replays up to the point you want to keep changes and sends replay confirmation.
+In other words there's no way to cut a sequence of changes out of the middle of
+the pending change stream.
+
+Once you've peeked the stream and know the LSN you want to discard up to, you
+can use `pg_logical_slot_peek_changes`, specifying an `upto_lsn`, to consume
+changes from the slot up to but not including that point, i.e. that will be the
+point at which replay resumes.
+
+For example, if you wanted to discard the first transaction in the example
+from the section above, i.e. discard xact 5562 and start decoding at xact
+5563 from its' BEGIN lsn `0/4E8ABA8`, you'd run:
+
+ SELECT location, xid, data
+ FROM pg_logical_slot_get_changes('demo_slot', '0/4E8ABA8', NULL,
+ 'min_proto_version', '1', 'max_proto_version', '1',
+ 'startup_params_format', '1', 'proto_format', 'json');
+
+Note that `_get_changes` is used instead of `_peek_changes` and that
+the `upto_lsn` is `'0/4E8ABA8'` instead of `NULL`.
+
+
+
+
+
diff --git a/contrib/pglogical_output/doc/.gitignore b/contrib/pglogical_output/doc/.gitignore
new file mode 100644
index 0000000..2874bff
--- /dev/null
+++ b/contrib/pglogical_output/doc/.gitignore
@@ -0,0 +1 @@
+protocol.html
diff --git a/contrib/pglogical_output/doc/DESIGN.md b/contrib/pglogical_output/doc/DESIGN.md
new file mode 100644
index 0000000..711539d
--- /dev/null
+++ b/contrib/pglogical_output/doc/DESIGN.md
@@ -0,0 +1,230 @@
+# Design decisions
+
+Explanations of why things are done the way they are.
+
+## Why does pglogical_output exist when there's wal2json etc?
+
+`pglogical_output` does plenty more than convert logical decoding change
+messages to a wire format and send them to the client.
+
+It handles format negotiations, sender-side filtering using pluggable hooks
+(and the associated plugin handling), etc. The protocol its self is also
+important, and incorporates elements like binary datum transfer that can't be
+easily or efficiently achieved with json.
+
+## Custom binary protocol
+
+Why do we have a custom binary protocol inside the walsender / copy both protocol,
+rather than using a json message representation?
+
+Speed and compactness. It's expensive to create json, with lots of allocations.
+It's expensive to decode it too. You can't represent raw binary in json, and must
+encode it, which adds considerable overhead for some data types. Using the
+obvious, easy to decode json representations also makes it difficult to do
+later enhancements planned for the protocol and decoder, like caching row
+metadata.
+
+The protocol implementation is fairly well encapsulated, so in future it should
+be possible to emit json instead for clients that request it. Right now that's
+not the priority as tools like wal2json already exist for that.
+
+## Column metadata
+
+The output plugin sends metadata for columns - at minimum, the column names -
+before each row that first refers to that relation.
+
+The reason metadata must be sent is that the upstream and downstream table's
+attnos don't necessarily correspond. The column names might, and their ordering
+might even be the same, but any column drop or column type change will result
+in a dropped column on one side. So at the user level the tables look the same,
+but their attnos don't match, and if we rely on attno for replication we'll get
+the wrong data in the wrong columns. Not pretty.
+
+That could be avoided by requiring that the downstream table be strictly
+maintained by DDL replication, but:
+
+* We don't want to require DDL replication
+* That won't work with multiple upstreams feeding into a table
+* The initial table creation still won't be correct if the table has dropped
+ columns, unless we (ab)use `pg_dump`'s `--binary-upgrade` support to emit
+ tables with dropped columns, which we don't want to do.
+
+So despite the bandwidth cost, we need to send metadata.
+
+Support for type metadata is penciled in to the protocol so that clients that
+don't have table definitions at all - like queueing engines - can decode the
+data. That'll also permit type validation sanity checking on the apply side
+with logical replication.
+
+The upstream expects the client to cache this metadata and re-use it when data
+is sent for the relation again. Cache size controls, an LRU and purge
+notifications will be added.
+
+## Relation metadata cache size controls
+
+The relation metadata cache will have downstream size control added. The
+downstream will send a parameter indicating that it supports caching, and the
+maximum cache size desired. The option will have settings for "no cache",
+"cache unlimited" and "fixed size LRU [size specified]".
+
+Since there is no downstream-to-upstream communication after the startup params
+there's no easy way for the downstream to tell the upstream when it purges
+cache entries. So the downstream cache is a slave cache that must depend
+strictly on the upstream cache. The downstream tells the upstream how to manage
+its cache and then after that it just follows orders.
+
+To keep the caches in sync so the upstream never sends a row without knowing
+the downstream has metadata for it cached the downstream must always cache
+relation metadata when it receives it, and may not purge it from its cache
+until it receives a purge message for that relation from the upstream. If a
+new metadata message for the same relation arrives it *must* replace the old
+entry in the cache.
+
+The downstream does *not* have to promptly purge or invalidate cache entries
+when it gets purge messages from the upstream. They are just notifications that
+the upstream no longer expects the downstream to retain that cache entry and
+will re-send it if it is required again later.
+
+## Not an extension
+
+There's no extension script for pglogical_output. That's by design. We've tried
+really hard to avoid needing one, allowing applications using pglogical_output
+to entirely define any SQL level catalogs they need and interact with them
+using the hooks.
+
+That way applications don't have to deal with some of their catalog data being
+in pglogical_output extension catalogs and some being in their own.
+
+There's no issue with dump and restore that way either. The app controls it
+entirely and pglogical_output doesn't need any policy or tools for it.
+
+pglogical_output is meant to be a re-usable component of other solutions. Users
+shouldn't need to care about it directly.
+
+## Hooks
+
+Quite a bit of functionality that could be done directly in the output
+plugin is instead delegated to pluggable hooks. Replication origin filtering
+for example.
+
+That's because pglogical_output tries hard not to know anything about the
+topology of the replication cluster and leave that to applications using the
+plugin. It doesn't
+
+
+## Hook entry point as a SQL function
+
+The hooks entry point is a SQL function that populates a passed `internal`
+struct with hook function pointers.
+
+The reason for this is that hooks are specified by a remote peer over the
+network. We can't just let the peer say "dlsym() this arbitrary function name
+and call it with these arguments" for fairly obvious security reasons. At bare
+minimum all replication using hooks would have to be superuser-only if we did
+that.
+
+The SQL entry point is only called once per decoding session and the rest of
+the calls are plain C function pointers.
+
+## The startup reply message
+
+The protocol design choices available to `pg_logical` are constrained by being
+contained in the copy-both protocol within the fe/be protocol, running as a
+logical decoding plugin. The plugin has no direct access to the network socket
+and can't send or receive messages whenever it wants, only under the control of
+the walsender and logical decoding framework.
+
+The only opportunity for the client to send data directly to the logical
+decoding plugin is in the `START_REPLICATION` parameters, and it can't send
+anything to the client before that point.
+
+This means there's no opportunity for a multi-way step negotiation between
+client and server. We have to do all the negotiation we're going to in a single
+exchange of messages - the setup parameters and then the replication start
+message. All the client can do if it doesn't like the offer the server makes is
+disconnect and try again with different parameters.
+
+That's what the startup message is for. It reports the plugin's capabilities
+and tells the client which requested options were honoured. This gives the
+client a chance to decide if it's happy with the output plugin's decision
+or if it wants to reconnect and try again with different options. Iterative
+negotiation, effectively.
+
+## Unrecognised parameters MUST be ignored by client and server
+
+To ensure upward and downward compatibility, the output plugin must ignore
+parameters set by the client if it doesn't recognise them, and the client
+must ignore parameters it doesn't recognise in the server's startup reply
+message.
+
+This ensures that older clients can talk to newer servers and vice versa.
+
+For this to work, the server must never enable new functionality such as
+protocol message types, row formats, etc without the client explicitly
+specifying via a startup parameter that it understands the new functionality.
+Everything must be negotiated.
+
+Similarly, a newer client talking to an older server may ask the server to
+enable functionality, but it can't assume the server will actually honour that
+request. It must check the server's startup reply message to see if the server
+confirmed that it enabled the requested functionality. It might choose to
+disconnect and report an error to the user if the server didn't do what it
+asked. This can be important, e.g. when a security-significant hook is
+specified.
+
+## Support for transaction streaming
+
+Presently logical decoding requires that a transaction has committed before it
+can *begin* sending it to the client. This means long running xacts can take 2x
+as long, since we can't start apply on the replica until the xact is committed
+on the master.
+
+Additionally, a big xact will cause large delays in apply of smaller
+transactions because logical decoding reoreders transactions into strict commit
+order and replays them in that sequence. Small transactions that commited after
+the big transaction cannot be replayed to the replica until the big transaction
+is transferred over the wire, and we can't get a head start on that while it's
+still running.
+
+Finally, the accumulation of a big transaction in the reorder buffer means that
+storage on the upstream must be sufficient to hold the entire transaction until
+it can be streamed to the replica and discarded. That is in addition to the
+copy in retained WAL, which cannot be purged until replay is confirmed past
+commit for that xact. The temporary copy serves no data safety purpose; it can
+be regenerated from retained WAL is just a spool file.
+
+There are big upsides to waiting until commit. Rolled-back transactions and
+subtransactions are never sent at all. The apply/downstream side is greatly
+simplified by not needing to do transaction ordering, worry about
+interdependencies and conflicts during apply. The commit timestamp is known
+from the beginning of replay, allowing for smarter conflict resolution
+behaviour in multi-master scenarios. Nonetheless sometimes we want to be able
+to stream changes in advance of commit.
+
+So we need the ability to start streaming a transaction from the upstream as
+its changes are seen in WAL, either applying it immediately on the downstream
+or spooling it on the downstream until it's committed. This requires changes
+to the logical decoding facilities themselves, it isn't something pglogical_output
+can do alone. However, we've left room in pglogical_output to support this
+when support is added to logical decoding:
+
+* Flags in most message types let us add fields if we need to, like a
+ HAS_XID flag and an extra field for the transaction ID so we can
+ differentiate between concurrent transactions when streaming. The space
+ isn't wasted the rest of the time.
+
+* The upstream isn't allowed to send new message types, etc, without a
+ capability flag being set by the client. So for interleaved xacts we won't
+ enable them in logical decoding unless the client tells us the client is
+ prepared to cope with them by sending additional startup parameters.
+
+Note that for consistency reasons we still have to commit things in the same
+order on the downstream. The purpose of transaction streaming is to reduce the
+latency between the commit of the last xact before a big one and the first xact
+after the big one, minimising the duration of the stall in the flow of smaller
+xacts perceptible on the downstream.
+
+Transaction streaming also makes parallel apply on the downstream possible,
+though it is not necessary to have parallel apply to benefit from transaction
+streaming. Parallel apply has further complexities that are outside the scope
+of the output plugin design.
diff --git a/contrib/pglogical_output/doc/protocol.txt b/contrib/pglogical_output/doc/protocol.txt
new file mode 100644
index 0000000..afe3bb3
--- /dev/null
+++ b/contrib/pglogical_output/doc/protocol.txt
@@ -0,0 +1,544 @@
+= Pg_logical protocol
+
+pglogical_output defines a libpq subprocotol for streaming tuples, metadata,
+etc, from the decoding plugin to receivers.
+
+This protocol is an inner layer in a stack:
+
+ * tcp or unix sockets
+ ** libpq protocol
+ *** libpq replication subprotocol (COPY BOTH etc)
+ **** pg_logical output plugin => consumer protocol
+
+so clients can simply use libpq's existing replication protocol support,
+directly or via their libpq-wrapper driver.
+
+This is a binary protocol intended for compact representation.
+
+`pglogical_output` also supports a json-based text protocol with json
+representations of the same changesets, supporting all the same hooks etc,
+intended mainly for tracing/debugging/diagnostics. That protocol is not
+discussed here.
+
+== ToC
+
+== Protocol flow
+
+The protocol flow is primarily from upstream walsender/decoding plugin to the
+downstream receiver.
+
+The only information the flows downstream-to-upstream is:
+
+ * The initial parameter list sent to `START_REPLICATION`; and
+ * replay progress messages
+
+We can accept an arbitrary list of params to `START_REPLICATION`. After
+that we have no general purpose channel for information to flow upstream. That
+means we can't do a multi-step negotiation/handshake for determining the
+replication options to use, binary protocol, etc.
+
+The main form of negotiation is the client getting a "take it or leave it" set
+of settings from the server in an initial startup message sent before any
+replication data (see below) and, if it doesn't like them, reconnecting with
+different startup options.
+
+Except for the negotiation via initial parameter list and then startup message
+the protocol flow is the same as any other walsender-based logical replication
+plugin. The data stream is sent in COPY BOTH mode as a series of CopyData
+messages encapsulating replication data, and ends when the client disconnects.
+There's no facility for ending the COPY BOTH mode and returning to the
+walsender command parser to issue new commands. This is a limiation of the
+walsender interface, not pglogical_output.
+
+== Protocol messages
+
+The individual protocol messages are discussed in the following sub-sections.
+Protocol flow and logic comes in the next major section.
+
+Absolutely all top-level protocol messages begin with a message type byte.
+While represented in code as a character, this is a signed byte with no
+associated encoding.
+
+Since the PostgreSQL libpq COPY protocol supplies a message length there’s no
+need for top-level protocol messages to embed a length in their header.
+
+=== BEGIN message
+
+A stream of rows starts with a `BEGIN` message. Rows may only be sent after a
+`BEGIN` and before a `COMMIT`.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**B**’ (0x42)
+|flags|uint8| * 0-3: Reserved, client _must_ ERROR if set and not recognised.
+|lsn|uint64|“final_lsn” in decoding context - currently it means lsn of commit
+|commit time|uint64|“commit_time” in decoding context
+|remote XID|uint32|“xid” in decoding context
+|===
+
+=== Forwarded transaction origin message
+
+The message after the `BEGIN` may be a _forwarded transaction origin_ message
+indicating what upstream node the transaction came from.
+
+Sent if the immediately prior message was a `BEGIN` message, the upstream
+transaction was forwarded from another node, and replication origin forwarding
+is enabled, i.e. `forward_changeset_origins` is `t` in the startup reply
+message.
+
+A "node" could be another host, another DB on the same host, or pretty much
+anything. Whatever origin name is found gets forwarded. The origin identifier
+is of arbitrary and application-defined format. Applications _should_ prefix
+their origin identifier with a fixed application name part, like `bdr_`,
+`myapp_`, etc. It is application-defined what an application does with
+forwarded transactions from other applications.
+
+An origin message with a zero-length origin name indicates that the origin
+could not be identified but was (probably) not the local node. It is
+client-defined what action is taken in this case.
+
+It is a protocol error to send/receive a forwarded transaction origin message
+at any time other than immediately after a `BEGIN` message.
+
+The origin identifier is typically closely related to replication slot names
+and replication origins’ names in an application system.
+
+For more detail see _Changeset Forwarding_ in the README.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**O**’ (0x4f)
+|flags|uint8| * 0-3: Reserved, application _must_ ERROR if set and not recognised
+|origin_lsn|uint64|Log sequence number (LSN, XLogRecPtr) of the transaction’s commit record on its origin node (as opposed to the forwarding node’s commit LSN, which is ‘lsn’ in the BEGIN message)
+|origin_identifier_length|uint8|Length in bytes of origin_identifier
+|origin_identifier|signed char[origin_identifier_length]|An origin identifier of arbitrary, upstream-application-defined structure. _Should_ be text in the same encoding as the upstream database. NULL-terminated. _Should_ be 7-bit ASCII.
+|===
+
+=== COMMIT message
+A stream of rows ends with a `COMMIT` message.
+
+There is no `ROLLBACK` message because aborted transactions are not sent by the
+upstream.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**C**’ (0x43)
+|Flags|uint8| * 0-3: Reserved, client _must_ ERROR if set and not recognised
+|Commit LSN|uint64|commit_lsn in decoding commit decode callback. This is the same value as in the BEGIN message, and marks the end of the transaction.
+|End LSN|uint64|end_lsn in decoding transaction context
+|Commit time|uint64|commit_time in decoding transaction context
+|===
+
+=== INSERT, UPDATE or DELETE message
+
+After a `BEGIN` or metadata message, the downstream should expect to receive
+zero or more row change messages, composed of an insert/update/delete message
+with zero or more tuple fields, each of which has one or more tuple field
+values.
+
+The row’s relidentifier _must_ match that of the most recently preceding
+metadata message. All consecutive row messages must currently have the same
+relidentifier. (_Later extensions to add metadata caching will relax these
+requirements for clients that advertise caching support; see the documentation
+on metadata messages for more detail_).
+
+It is an error to decode rows using metadata received after the row was
+received, or using metadata that is not the most recently received metadata
+revision that still predates the row. I.e. in the sequence M1, R1, R2, M2, R3,
+M4: R1 and R2 must be decoded using M1, and R3 must be decoded using M2. It is
+an error to use M4 to decode any of the rows, to use M1 to decode R3, or to use
+M2 to decode R1 and R2.
+
+Row messages _may not_ arrive except during a transaction as delimited by `BEGIN`
+and `COMMIT` messages. It is an error to receive a row message outside a
+transaction.
+
+Any unrecognised tuple type or tuple part type is an error on the downstream
+that must result in a client disconnect and error message. Downstreams are
+expected to negotiate compatibility, and upstreams must not add new tuple types
+or tuple field types without negotiation.
+
+The downstream reads rows until the next non-row message is received. There is
+no other end marker or any indication of how many rows to expect in a sequence.
+
+==== Row message header
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**I**’nsert (0x49), ‘**U**’pdate’ (0x55) or ‘**D**’elete (0x44)
+|flags|uint8|Row flags (reserved)
+|relidentifier|uint32|relidentifier that matches the table metadata message sent for this row.
+(_Not present in BDR, which sends nspname and relname instead_)
+|[tuple parts]|[composite]|
+|===
+
+One or more tuple-parts fields follow.
+
+==== Tuple fields
+
+|===
+|Tuple type|signed char|Identifies the kind of tuple being sent.
+
+|tupleformat|signed char|‘**T**’ (0x54)
+|natts|uint16|Number of fields sent in this tuple part.
+(_Present in BDR, but meaning significantly different here)_
+|[tuple field values]|[composite]|
+|===
+
+===== Tuple tupleformat compatibility
+
+Unrecognised _tupleformat_ kinds are a protocol error for the downstream.
+
+==== Tuple field value fields
+
+These message parts describe individual fields within a tuple.
+
+There are two kinds of tuple value fields, abbreviated and full. Which is being
+read is determined based on the first field, _kind_.
+
+Abbreviated tuple value fields are nothing but the message kind:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|kind|signed char| * ‘**n**’ull (0x6e) field
+|===
+
+Full tuple value fields have a length and datum:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|kind|signed char| * ‘**i**’nternal binary (0x62) field
+|length|int4|Only defined for kind = i\|b\|t
+|data|[length]|Data in a format defined by the table metadata and column _kind_.
+|===
+
+===== Tuple field values kind compatibility
+
+Unrecognised field _kind_ values are a protocol error for the downstream. The
+downstream may not continue processing the protocol stream after this
+point**.**
+
+The upstream may not send ‘**i**’nternal or ‘**b**’inary format values to the
+downstream without the downstream negotiating acceptance of such values. The
+downstream will also generally negotiate to receive type information to use to
+decode the values. See the section on startup parameters and the startup
+message for details.
+
+=== Table/row metadata messages
+
+Before sending changed rows for a relation, a metadata message for the relation
+must be sent so the downstream knows the namespace, table name, column names,
+optional column types, etc. A relidentifier field, an arbitrary numeric value
+unique for that relation on that upstream connection, maps the metadata to
+following rows.
+
+A client should not assume that relation metadata will be followed immediately
+(or at all) by rows, since future changes may lead to metadata messages being
+delivered at other times. Metadata messages may arrive during or between
+transactions.
+
+The upstream may not assume that the downstream retains more metadata than the
+one most recent table metadata message. This applies across all tables, so a
+client is permitted to discard metadata for table x when getting metadata for
+table y. The upstream must send a new metadata message before sending rows for
+a different table, even if that metadata was already sent in the same session
+or even same transaction. _This requirement will later be weakened by the
+addition of client metadata caching, which will be advertised to the upstream
+with an output plugin parameter._
+
+Columns in metadata messages are numbered from 0 to natts-1, reading
+consecutively from start to finish. The column numbers do not have to be a
+complete description of the columns in the upstream relation, so long as all
+columns that will later have row values sent are described. The upstream may
+choose to omit columns it doesn’t expect to send changes for in any given
+series of rows. Column numbers are not necessarily stable across different sets
+of metadata for the same table, even if the table hasn’t changed structurally.
+
+A metadata message may not be used to decode rows received before that metadata
+message.
+
+==== Table metadata header
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|Literal ‘**R**’ (0x52)
+|flags|uint8| * 0-6: Reserved, client _must_ ERROR if set and not recognised.
+|relidentifier|uint32|Arbitrary relation id, unique for this upstream. In practice this will probably be the upstream table’s oid, but the downstream can’t assume anything.
+|nspnamelength|uint8|Length of namespace name
+|nspname|signed char[nspnamelength]|Relation namespace (null terminated)
+|relnamelength|uint8|Length of relation name
+|relname|char[relname]|Relation name (null terminated)
+|attrs block|signed char|Literal: ‘**A**’ (0x41)
+|natts|uint16|number of attributes
+|[fields]|[composite]|Sequence of ‘natts’ column metadata blocks, each of which begins with a column delimiter followed by zero or more column metadata blocks, each with the same column metadata block header.
+
+This chunked format is used so that new metadata messages can be added without breaking existing clients.
+|===
+
+==== Column delimiter
+
+Each column’s metadata begins with a column metadata header. This comes
+immediately after the natts field in the table metadata header or after the
+last metadata block in the prior column.
+
+It has the same char header as all the others, and the flags field is the same
+size as the length field in other blocks, so it’s safe to read this as a column
+metadata block header.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|blocktype|signed char|‘**C**’ (0x43) - column
+|flags|uint8|Column info flags
+|===
+
+==== Column metadata block header
+
+All column metadata blocks share the same header, which is the same length as a
+column delimiter:
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|blocktype|signed char|Identifies the kind of metadata block that follows.
+|blockbodylength|uint16|Length of block in bytes, excluding blocktype char and length field.
+|===
+
+==== Column name block
+
+This block just carries the name of the column, nothing more. It begins with a
+column metadata block, and the rest of the message is the column name.
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|[column metadata block header]|[composite]|blocktype = ‘**N**’ (0x4e)
+|colname|char[blockbodylength]|Column name.
+|===
+
+
+==== Column type block
+
+T.B.D.
+
+Not defined in first protocol revision.
+
+Likely to send a type identifier (probably the upstream oid) as a reference to
+a “type info” protocol message to be delivered before. Then we can cache the
+type descriptions and avoid repeating long schemas and names, just using the
+oids.
+
+Needs to have room to handle:
+
+ * built-in core types
+ * extension types (ext version may vary)
+ * enum types (CREATE TYPE … AS ENUM)
+ * range types (CREATE TYPE … AS RANGE)
+ * composite types (CREATE TYPE … AS (...))
+ * custom types (CREATE TYPE ( input = x_in, output = x_out ))
+
+… some of which can be nested
+
+== Startup message
+
+After processing output plugin arguments, the upstream output plugin must send
+a startup message as its first message on the wire. It is a trivial header
+followed by alternating key and value strings represented as null-terminated
+unsigned char strings.
+
+This message specifies the capabilities the output plugin enabled and describes
+the upstream server and plugin. This may change how the client decodes the data
+stream, and/or permit the client to disconnect and report an error to the user
+if the result isn’t acceptable.
+
+If replication is rejected because the client is incompatible or the server is
+unable to satisfy required options, the startup message may be followed by a
+libpq protocol FATAL message that terminates the session. See “Startup errors”
+below.
+
+The parameter names and values are sent as alternating key/value pairs as
+null-terminated strings, e.g.
+
++“key1\0parameter1\0key2\0value2\0”+
+
+|===
+|*Message*|*Type/Size*|*Notes*
+
+|Message type|signed char|‘**S**’ (0x53) - startup
+|Startup message version|uint8|Value is always “1”.
+|(parameters)|null-terminated key/value pairs|See table below for parameter definitions.
+|===
+
+=== Startup message parameters
+
+Since all parameter values are sent as strings, the value types given below specify what the value must be reasonably interpretable as.
+
+|===
+|*Key name*|*Value type*|*Description*
+
+|max_proto_version|integer|Newest version of the protocol supported by output plugin.
+|min_proto_version|integer|Oldest protocol version supported by server.
+|proto_format|text|Protocol format requested. native (documented here) or json. Default is native.
+|coltypes|boolean|Column types will be sent in table metadata.
+|pg_version_num|integer|PostgreSQL server_version_num of server, if it’s PostgreSQL. e.g. 090400
+|pg_version|string|PostgreSQL server_version of server, if it’s PostgreSQL.
+|pg_catversion|uint32|Version of the PostgreSQL system catalogs on the upstream server, if it’s PostgreSQL.
+|binary|_set of parameters, specified separately_|See “_the __‘binary’__ parameters_” below, and “_Parameters relating to exchange of binary values_”
+|database_encoding|string|The native text encoding of the database the plugin is running in
+|encoding|string|Field values for textual data will be in this encoding in native protocol text, binary or internal representation. For the native protocol this is currently always the same as `database_encoding`. For text-mode json protocol this is always the same as `client_encoding`.
+|forward_changeset_origins|bool|Tells the client that the server will send changeset origin information. See “_Changeset forwarding_” for details.
+|no_txinfo|bool|Requests that variable transaction info such as XIDs, LSNs, and timestamps be omitted from output. Mainly for tests. Currently ignored for protos other than json.
+|===
+
+
+The ‘binary’ parameter set:
+==
+|===
+|*Key name*|*Value type*|*Description*
+
+|binary.internal_basetypes|boolean|If true, PostgreSQL internal binary representations for row field data may be used for some or all row fields, if here the type is appropriate and the binary compatibility parameters of upstream and downstream match. See binary.want_internal_basetypes in the output plugin parameters for details.
+
+May only be true if _binary.want_internal_basetypes_ was set to true by the client in the parameters and the client’s accepted binary format matches that of the server.
+|binary.binary_basetypes|boolean|If true, external binary format (send/recv format) may be used for some or all row field data where the field type is a built-in base type whose send/recv format is compatible with binary.binary_pg_version .
+
+May only be set if _binary.want_binary_basetypes_ was set to true by the client in the parameters and the client’s accepted send/recv format matches that of the server.
+|binary.binary_pg_version|uint16|The PostgreSQL major version that send/recv format values will be compatible with. This is not necessarily the actual upstream PostgreSQL version.
+|binary.sizeof_int|uint8|sizeof(int) on the upstream.
+|binary.sizeof_long|uint8|sizeof(long) on the upstream.
+|binary.sizeof_datum|uint8|Same as sizeof_int, but for the PostgreSQL Datum typedef.
+|binary.maxalign|uint8|Upstream PostgreSQL server’s MAXIMUM_ALIGNOF value - platform dependent, determined at build time.
+|binary.bigendian|bool|True iff the upstream is big-endian.
+|binary.float4_byval|bool|Upstream PostgreSQL’s float4_byval compile option.
+|binary.float8_byval|bool|Upstream PostgreSQL’s float8_byval compile option.
+|binary.integer_datetimes|bool|Whether TIME, TIMESTAMP and TIMESTAMP WITH TIME ZONE will be sent using integer or floating point representation.
+
+Usually this is the value of the upstream PostgreSQL’s integer_datetimes compile option.
+|===
+== Startup errors
+
+If the server rejects the client’s connection - due to non-overlapping protocol
+support, unrecognised parameter formats, unsupported required parameters like
+hooks, etc - then it will follow the startup reply message with a
++++<u>+++normal libpq protocol error message+++</u>+++. (Current versions send
+this before the startup message).
+
+== Arguments client supplies to output plugin
+
+The one opportunity for the downstream client to send information (other than replay feedback) to the upstream is at connect-time, as an array of arguments to the output plugin supplied to START LOGICAL REPLICATION.
+
+There is no back-and-forth, no handshake.
+
+As a result, the client mainly announces capabilities and makes requests of the output plugin. The output plugin will ERROR if required parameters are unset, or where incompatibilities that cannot be resolved are found. Otherwise the output plugin reports what it could and could not honour in the startup message it sends as the first message on the wire down to the client. The client chooses whether to continue replay or to disconnect and report an error to the user, then possibly reconnect with different options.
+
+=== Output plugin arguments
+
+The output plugin’s key/value arguments are specified in pairs, as key and value. They’re what’s passed to START_REPLICATION, etc.
+
+All parameters are passed in text form. They _should_ be limited to 7-bit ASCII, since the server’s text encoding is not known, but _may_ be normalized precomposed UTF-8. The types specified for parameters indicate what the output plugin should attempt to convert the text into. Clients should not send text values that are outside the range for that type.
+
+==== Capabilities
+
+Many values are capabilities flags for the client, indicating that it understands optional features like metadata caching, binary format transfers, etc. In general the output plugin _may_ disregard capabilities the client advertises as supported and act as if they are not supported. If a capability is advertised as unsupported or is not advertised the output plugin _must not_ enable the corresponding features.
+
+In other words, don’t send the client something it’s not expecting.
+
+==== Protocol versioning
+
+Two parameters max_proto_version and min_proto_version, which clients must always send, allow negotiation of the protocol version. The output plugin must ERROR if the client protocol support does not overlap its own protocol support range.
+
+The protocol version is only incremented when there are major breaking changes that all or most clients must be modified to accommodate. Most changes are done by adding new optional messages and/or by having clients advertise capabilities to opt in to features.
+
+Because these versions are expected to be incremented, to make it clear that the format of the startup parameters themselves haven’t changed, the first key/value pair _must_ be the parameter startup_params_format with value “1”.
+
+|===
+|*Key*|*Type*|*Value(s)*|*Notes*
+
+|startup_params_format|int8|1|The format version of this startup parameter set. Always the digit 1 (0x31), null terminated.
+|max_proto_version|int32|1|Newest version of the protocol supported by client. Output plugin must ERROR if supported version too old. *Required*, ERROR if missing.
+|min_proto_version|int32|1|Oldest version of the protocol supported by client. Output plugin must ERROR if supported version too old. *Required*, ERROR if missing.
+|===
+
+==== Client requirements and capabilities
+
+|===
+|*Key*|*Type*|*Default*|*Notes*
+
+|expected_encoding|string|null|The text encoding the downstream expects field values to be in. Applies to text, binary and internal representations of field values in native format. Has no effect on other protocol content. If specified, the upstream must honour it. For json protocol, must be unset or match `client_encoding`. (Current plugin versions ERROR if this is set for the native protocol and not equal to the upstream database's encoding).
+|want_coltypes|boolean|false|The client wants to receive data type information about columns.
+|===
+
+==== General client information
+
+These keys tell the output plugin about the client. They’re mainly for informational purposes. In particular, the versions must _not_ be used to determine compatibility for binary or send/recv format, as non-PostgreSQL clients will simply not send them at all but may still understand binary or send/recv format fields.
+
+|===
+|*Key*|*Type*|*Default*|*Notes*
+
+|pg_version_num|integer|null|PostgreSQL server_version_num of client, if it’s PostgreSQL. e.g. 090400
+|pg_version|string|null|PostgreSQL server_version of client, if it’s PostgreSQL.
+|===
+
+
+==== Parameters relating to exchange of binary values
+
+The downstream may specify to the upstream that it is capable of understanding binary (PostgreSQL internal binary datum format), and/or send/recv (PostgreSQL binary interchange) format data by setting the binary.want_binary_basetypes and/or binary.want_internal_basetypes options, or other yet-to-be-defined options.
+
+An upstream output plugin that does not support one or both formats _may_ ignore the downstream’s binary support and send text format, in which case it may ignore all binary. parameters. All downstreams _must_ support text format. An upstream output plugin _must not_ send binary or send/recv format unless the downstream has announced it can receive it. If both upstream and downstream support both formats an upstream should prefer binary format and fall back to send/recv, then to text, if compatibility requires.
+
+Internal and binary format selection should be done on a type-by-type basis. It is quite normal to send ‘text’ format for extension types while sending binary for built-in types.
+
+The downstream _must_ specify its compatibility requirements for internal and binary data if it requests either or both formats. The upstream _must_ honour these by falling back from binary to send/recv, and from send/recv to text, where the upstream and downstream are not compatible.
+
+An unspecified compatibility field _must_ presumed to be unsupported by the downstream so that older clients that don’t know about a change in a newer version don’t receive unexpected data. For example, in the unlikely event that PostgreSQL 99.8 switched to 128-bit DPD (Densely Packed Decimal) representations of NUMERIC instead of the current arbitrary-length BCD (Binary Coded Decimal) format, a new binary.dpd_numerics parameter would be added. Clients that didn’t know about the change wouldn’t know to set it, so the upstream would presume it unsupported and send text format NUMERIC to those clients. This also means that clients that support the new format wouldn’t be able to receive the old format in binary from older servers since they’d specify dpd_numerics = true in their compatibility parameters.
+
+At this time a downstream may specify compatibility with only one value for a given option; i.e. a downstream cannot say it supports both 4-byte and 8-byte sizeof(int). Leaving it unspecified means the upstream must assume the downstream supports neither. (A future protocol extension may allow clients to specify alternative sets of supported formats).
+
+The `pg_version` option _must not_ be used to decide compatibility. Use `binary.basetypes_major_version` instead.
+
+|===
+|*Key name*|*Value type*|*Default*|*Description*
+
+|binary.want_binary_basetypes|boolean|false|True if the client accepts binary interchange (send/recv) format rows for PostgreSQL built-in base types.
+|binary.want_internal_basetypes|boolean|false|True if the client accepts PostgreSQL internal-format binary output for base PostgreSQL types not otherwise specified elsewhere.
+|binary.basetypes_major_version|uint16|null|The PostgreSQL major version (x.y) the downstream expects binary and send/recv format values to be in. Represented as an integer in XXYY format (no leading zero since it’s an integer), e.g. 9.5 is 905. This corresponds to PG_VERSION_NUM/100 in PostgreSQL.
+|binary.sizeof_int|uint8|+null+|sizeof(int) on the downstream.
+|binary.sizeof_long|uint8|null|sizeof(long) on the downstream.
+|binary.sizeof_datum|uint8|null|Same as sizeof_int, but for the PostgreSQL Datum typedef.
+|binary.maxalign|uint8|null|Downstream PostgreSQL server’s maxalign value - platform dependent, determined at build time.
+|binary.bigendian|bool|null|True iff the downstream is big-endian.
+|binary.float4_byval|bool|null|Downstream PostgreSQL’s float4_byval compile option.
+|binary.float8_byval|bool|null|Downstream PostgreSQL’s float8_byval compile option.
+|binary.integer_datetimes|bool|null|Downstream PostgreSQL’s integer_datetimes compile option.
+|===
+
+== Extensibility
+
+Because of the use of optional parameters in output plugin arguments, and the
+confirmation/response sent in the startup packet, a basic handshake is possible
+between upstream and downstream, allowing negotiation of capabilities.
+
+The output plugin must never send non-optional data or change its wire format
+without confirmation from the client that it can understand the new data. It
+may send optional data without negotiation.
+
+When extending the output plugin arguments, add-ons are expected to prefix all
+keys with the extension name, and should preferably use a single top level key
+with a json object value to carry their extension information. Additions to the
+startup message should follow the same pattern.
+
+Hooks and plugins can be used to add functionality specific to a client.
+
+== JSON protocol
+
+If `proto_format` is set to `json` then the output plugin will emit JSON
+instead of the custom binary protocol. JSON support is intended mainly for
+debugging and diagnostics.
+
+The JSON format supports all the same hooks.
diff --git a/contrib/pglogical_output/expected/basic_json.out b/contrib/pglogical_output/expected/basic_json.out
new file mode 100644
index 0000000..c48948e
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_json.out
@@ -0,0 +1,140 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+---------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "t"
+ hooks.row_filter_enabled | "f"
+ hooks.shutdown_hook_enabled | "f"
+ hooks.startup_hook_enabled | "f"
+ hooks.transaction_filter_enabled | "f"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+ pglogical_output_version | "10000"
+ relmeta_cache_size | "0"
+(20 rows)
+
+SELECT * FROM get_queued_data();
+ data
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "textval", "jsb": null, "seq": 1}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": "\\xdeadbeef0001", "js": null, "ts": null, "tx": null, "jsb": null, "seq": 2}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": "2045-09-12T12:34:56", "tx": "blah", "jsb": null, "seq": 3}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": {"key": "value"}, "ts": null, "tx": null, "jsb": {"key": "value"}, "seq": 4}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row1", "jsb": null, "seq": 6}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row2", "jsb": null, "seq": 7}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row3", "jsb": null, "seq": 8}, "relation": ["public", "demo"]}
+ {"action": "D", "oldtuple": {"ba": null, "js": null, "ts": null, "tx": null, "jsb": null, "seq": 7}, "relation": ["public", "demo"]}
+ {"action": "U", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "updated", "jsb": null, "seq": 6}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "I", "newtuple": {"id": 42}, "relation": ["public", "cat_test"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "1", "jsb": null, "seq": 9}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "2", "jsb": null, "seq": 10}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(28 rows)
+
+TRUNCATE TABLE json_decoding_output;
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/basic_json_1.out b/contrib/pglogical_output/expected/basic_json_1.out
new file mode 100644
index 0000000..85da990
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_json_1.out
@@ -0,0 +1,139 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+--------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "f"
+ hooks.row_filter_enabled | "f"
+ hooks.shutdown_hook_enabled | "f"
+ hooks.startup_hook_enabled | "f"
+ hooks.transaction_filter_enabled | "f"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+ relmeta_cache_size | "0"
+(19 rows)
+
+SELECT * FROM get_queued_data();
+ data
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "textval", "jsb": null, "seq": 1}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": "\\xdeadbeef0001", "js": null, "ts": null, "tx": null, "jsb": null, "seq": 2}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": "2045-09-12T12:34:56", "tx": "blah", "jsb": null, "seq": 3}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": {"key": "value"}, "ts": null, "tx": null, "jsb": {"key": "value"}, "seq": 4}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row1", "jsb": null, "seq": 6}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row2", "jsb": null, "seq": 7}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "row3", "jsb": null, "seq": 8}, "relation": ["public", "demo"]}
+ {"action": "D", "oldtuple": {"ba": null, "js": null, "ts": null, "tx": null, "jsb": null, "seq": 7}, "relation": ["public", "demo"]}
+ {"action": "U", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "updated", "jsb": null, "seq": 6}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "I", "newtuple": {"id": 42}, "relation": ["public", "cat_test"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "1", "jsb": null, "seq": 9}, "relation": ["public", "demo"]}
+ {"action": "I", "newtuple": {"ba": null, "js": null, "ts": null, "tx": "2", "jsb": null, "seq": 10}, "relation": ["public", "demo"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(28 rows)
+
+TRUNCATE TABLE json_decoding_output;
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/basic_native.out b/contrib/pglogical_output/expected/basic_native.out
new file mode 100644
index 0000000..e9d7a6e
--- /dev/null
+++ b/contrib/pglogical_output/expected/basic_native.out
@@ -0,0 +1,113 @@
+\i sql/basic_setup.sql
+SET synchronous_commit = on;
+-- Schema setup
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Queue up some work to decode with a variety of types
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ count
+-------
+ 39
+(1 row)
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+ count
+-------
+ 39
+(1 row)
+
+-- Now enable the relation metadata cache and verify that we get the expected
+-- reduction in number of messages. Not much else we can look for.
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'relmeta_cache_size', '-1');
+ count
+-------
+ 29
+(1 row)
+
+\i sql/basic_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/expected/cleanup.out b/contrib/pglogical_output/expected/cleanup.out
new file mode 100644
index 0000000..e7a02c8
--- /dev/null
+++ b/contrib/pglogical_output/expected/cleanup.out
@@ -0,0 +1,4 @@
+DROP TABLE excluded_startup_keys;
+DROP TABLE json_decoding_output;
+DROP FUNCTION get_queued_data();
+DROP FUNCTION get_startup_params();
diff --git a/contrib/pglogical_output/expected/encoding_json.out b/contrib/pglogical_output/expected/encoding_json.out
new file mode 100644
index 0000000..82c719a
--- /dev/null
+++ b/contrib/pglogical_output/expected/encoding_json.out
@@ -0,0 +1,59 @@
+SET synchronous_commit = on;
+-- This file doesn't share common setup with the native tests,
+-- since it's specific to how the text protocol handles encodings.
+CREATE TABLE enctest(blah text);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+SET client_encoding = 'UTF-8';
+INSERT INTO enctest(blah)
+VALUES
+('áàä'),('fl'), ('½⅓'), ('カンジ');
+RESET client_encoding;
+SET client_encoding = 'LATIN-1';
+-- Will ERROR, explicit encoding request doesn't match client_encoding
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: expected_encoding must be unset or match client_encoding in text protocols
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Will succeed since we don't request any encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: character with byte sequence 0xef 0xac 0x82 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+-- Will succeed since it matches the current encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'LATIN-1',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+ERROR: character with byte sequence 0xef 0xac 0x82 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+RESET client_encoding;
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE enctest;
diff --git a/contrib/pglogical_output/expected/extension.out b/contrib/pglogical_output/expected/extension.out
new file mode 100644
index 0000000..af5cda4
--- /dev/null
+++ b/contrib/pglogical_output/expected/extension.out
@@ -0,0 +1,14 @@
+CREATE EXTENSION pglogical_output;
+SELECT pglogical_output_proto_version();
+ pglogical_output_proto_version
+--------------------------------
+ 1
+(1 row)
+
+SELECT pglogical_output_min_proto_version();
+ pglogical_output_min_proto_version
+------------------------------------
+ 1
+(1 row)
+
+DROP EXTENSION pglogical_output;
diff --git a/contrib/pglogical_output/expected/hooks_json.out b/contrib/pglogical_output/expected/hooks_json.out
new file mode 100644
index 0000000..1d20f5e
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_json.out
@@ -0,0 +1,204 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Test table filter
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+---------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "t"
+ hooks.row_filter_enabled | "t"
+ hooks.shutdown_hook_enabled | "t"
+ hooks.startup_hook_enabled | "t"
+ hooks.transaction_filter_enabled | "t"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+ pglogical_output_version | "10000"
+ relmeta_cache_size | "0"
+(20 rows)
+
+SELECT * FROM get_queued_data();
+ data
+---------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "U", "newtuple": {"id": 500}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(25 rows)
+
+-- test action filter
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output (ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+---------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "t"
+ hooks.row_filter_enabled | "t"
+ hooks.shutdown_hook_enabled | "t"
+ hooks.startup_hook_enabled | "t"
+ hooks.transaction_filter_enabled | "t"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+ pglogical_output_version | "10000"
+ relmeta_cache_size | "0"
+(20 rows)
+
+SELECT * FROM get_queued_data();
+ data
+--------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_filter"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(36 rows)
+
+TRUNCATE TABLE json_decoding_output;
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/hooks_json_1.out b/contrib/pglogical_output/expected/hooks_json_1.out
new file mode 100644
index 0000000..6e2aa5f
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_json_1.out
@@ -0,0 +1,202 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Test table filter
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+--------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "f"
+ hooks.row_filter_enabled | "t"
+ hooks.shutdown_hook_enabled | "t"
+ hooks.startup_hook_enabled | "t"
+ hooks.transaction_filter_enabled | "t"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+ relmeta_cache_size | "0"
+(19 rows)
+
+SELECT * FROM get_queued_data();
+ data
+---------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "U", "newtuple": {"id": 500}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(25 rows)
+
+-- test action filter
+TRUNCATE TABLE json_decoding_output;
+INSERT INTO json_decoding_output (ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+SELECT * FROM get_startup_params();
+ key | value
+----------------------------------+--------
+ binary.binary_basetypes | "f"
+ binary.float4_byval | "t"
+ binary.float8_byval | "t"
+ binary.internal_basetypes | "f"
+ binary.sizeof_datum | "8"
+ binary.sizeof_int | "4"
+ binary.sizeof_long | "8"
+ coltypes | "f"
+ database_encoding | "UTF8"
+ encoding | "UTF8"
+ forward_changeset_origins | "f"
+ hooks.row_filter_enabled | "t"
+ hooks.shutdown_hook_enabled | "t"
+ hooks.startup_hook_enabled | "t"
+ hooks.transaction_filter_enabled | "t"
+ max_proto_version | "1"
+ min_proto_version | "1"
+ no_txinfo | "t"
+ relmeta_cache_size | "0"
+(19 rows)
+
+SELECT * FROM get_queued_data();
+ data
+--------------------------------------------------------------------------------
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_filter"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_filter"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "I", "newtuple": {"id": 1}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 2}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 3}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 4}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 5}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 6}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 7}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 8}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 9}, "relation": ["public", "test_nofilt"]}
+ {"action": "I", "newtuple": {"id": 10}, "relation": ["public", "test_nofilt"]}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "f"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+ {"action": "B", "has_catalog_changes": "t"}
+ {"action": "C"}
+(36 rows)
+
+TRUNCATE TABLE json_decoding_output;
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/hooks_native.out b/contrib/pglogical_output/expected/hooks_native.out
new file mode 100644
index 0000000..4a547cb
--- /dev/null
+++ b/contrib/pglogical_output/expected/hooks_native.out
@@ -0,0 +1,104 @@
+\i sql/hooks_setup.sql
+CREATE EXTENSION pglogical_output_plhooks;
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
+-- Regular hook setup
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+ count
+-------
+ 40
+(1 row)
+
+-- Test action filter
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+ count
+-------
+ 53
+(1 row)
+
+-- Invalid row fiter hook function
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+ERROR: function public.nosuchfunction(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Hook filter functoin with wrong signature
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+ERROR: function public.wrong_signature_fn(regclass, "char", text) does not exist
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+\i sql/hooks_teardown.sql
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/expected/params_native.out b/contrib/pglogical_output/expected/params_native.out
new file mode 100644
index 0000000..8e0bc7d
--- /dev/null
+++ b/contrib/pglogical_output/expected/params_native.out
@@ -0,0 +1,133 @@
+SET synchronous_commit = on;
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+--
+-- Various invalid parameter combos:
+--
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: logical decoding output plugin "pglogical_output" produces binary output, but function "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+ERROR: client sent startup parameters in format 2 but we only support format 1
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: unrecognised encoding name bork passed to expected_encoding
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+ERROR: client requested protocol invalid but only "json" or "native" are supported
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+ data
+------
+(0 rows)
+
+-- relmeta cache with fixed size (not supported yet, so error)
+SELECT count(data) FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'relmeta_cache_size', '200');
+INFO: fixed size cache not supported, forced to off
+DETAIL: only relmeta_cache_size=0 (off) or relmeta_cache_size=-1 (unlimited) supported
+ count
+-------
+ 0
+(1 row)
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
diff --git a/contrib/pglogical_output/expected/params_native_1.out b/contrib/pglogical_output/expected/params_native_1.out
new file mode 100644
index 0000000..31d9486
--- /dev/null
+++ b/contrib/pglogical_output/expected/params_native_1.out
@@ -0,0 +1,134 @@
+SET synchronous_commit = on;
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+ ?column?
+----------
+ init
+(1 row)
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+--
+-- Various invalid parameter combos:
+--
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: logical decoding output plugin "pglogical_output" produces binary output, but "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ERROR: client sent min_proto_version=2 but we only support protocol 1 or lower
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+ERROR: client sent startup parameters in format 2 but we only support format 1
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ERROR: unrecognised encoding name bork passed to expected_encoding
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+ data
+------
+(0 rows)
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+ERROR: client requested protocol invalid but only "json" or "native" are supported
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+ data
+------
+(0 rows)
+
+-- relmeta cache with fixed size (not supported yet, so error)
+SELECT count(data) FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'relmeta_cache_size', '200');
+INFO: fixed size cache not supported, forced to off
+DETAIL: only relmeta_cache_size=0 (off) or relmeta_cache_size=-1 (unlimited) supported
+CONTEXT: slot "regression_slot", output plugin "pglogical_output", in the startup callback
+ count
+-------
+ 0
+(1 row)
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+ ?column?
+----------
+ drop
+(1 row)
+
diff --git a/contrib/pglogical_output/expected/prep.out b/contrib/pglogical_output/expected/prep.out
new file mode 100644
index 0000000..6501951
--- /dev/null
+++ b/contrib/pglogical_output/expected/prep.out
@@ -0,0 +1,26 @@
+CREATE TABLE excluded_startup_keys (key_name text primary key);
+INSERT INTO excluded_startup_keys
+VALUES
+('pg_version_num'),('pg_version'),('pg_catversion'),('binary.basetypes_major_version'),('binary.integer_datetimes'),('binary.bigendian'),('binary.maxalign'),('binary.binary_pg_version'),('sizeof_int'),('sizeof_long'),('sizeof_datum');
+CREATE UNLOGGED TABLE json_decoding_output(ch jsonb, rn integer);
+CREATE OR REPLACE FUNCTION get_startup_params()
+RETURNS TABLE ("key" text, "value" jsonb)
+LANGUAGE sql
+AS $$
+SELECT key, value
+FROM json_decoding_output
+CROSS JOIN LATERAL jsonb_each(ch -> 'params')
+WHERE rn = 1
+ AND key NOT IN (SELECT * FROM excluded_startup_keys)
+ AND ch ->> 'action' = 'S'
+ORDER BY key;
+$$;
+CREATE OR REPLACE FUNCTION get_queued_data()
+RETURNS TABLE (data jsonb)
+LANGUAGE sql
+AS $$
+SELECT ch
+FROM json_decoding_output
+WHERE rn > 1
+ORDER BY rn ASC;
+$$;
diff --git a/contrib/pglogical_output/pglogical_config.c b/contrib/pglogical_output/pglogical_config.c
new file mode 100644
index 0000000..a9c9921
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.c
@@ -0,0 +1,526 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_config.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_config.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output/compat.h"
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+
+#include "catalog/catversion.h"
+#include "catalog/namespace.h"
+
+#include "mb/pg_wchar.h"
+
+#include "nodes/makefuncs.h"
+
+#include "utils/builtins.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+typedef enum PGLogicalOutputParamType
+{
+ OUTPUT_PARAM_TYPE_BOOL,
+ OUTPUT_PARAM_TYPE_UINT32,
+ OUTPUT_PARAM_TYPE_INT32,
+ OUTPUT_PARAM_TYPE_STRING,
+ OUTPUT_PARAM_TYPE_QUALIFIED_NAME
+} PGLogicalOutputParamType;
+
+/* param parsing */
+static Datum get_param_value(DefElem *elem, bool null_ok,
+ PGLogicalOutputParamType type);
+
+static Datum get_param(List *options, const char *name, bool missing_ok,
+ bool null_ok, PGLogicalOutputParamType type,
+ bool *found);
+static bool parse_param_bool(DefElem *elem);
+static uint32 parse_param_uint32(DefElem *elem);
+static int32 parse_param_int32(DefElem *elem);
+
+static void
+process_parameters_v1(List *options, PGLogicalOutputData *data);
+
+enum {
+ PARAM_UNRECOGNISED,
+ PARAM_MAX_PROTOCOL_VERSION,
+ PARAM_MIN_PROTOCOL_VERSION,
+ PARAM_PROTOCOL_FORMAT,
+ PARAM_EXPECTED_ENCODING,
+ PARAM_BINARY_BIGENDIAN,
+ PARAM_BINARY_SIZEOF_DATUM,
+ PARAM_BINARY_SIZEOF_INT,
+ PARAM_BINARY_SIZEOF_LONG,
+ PARAM_BINARY_FLOAT4BYVAL,
+ PARAM_BINARY_FLOAT8BYVAL,
+ PARAM_BINARY_INTEGER_DATETIMES,
+ PARAM_BINARY_WANT_INTERNAL_BASETYPES,
+ PARAM_BINARY_WANT_BINARY_BASETYPES,
+ PARAM_BINARY_BASETYPES_MAJOR_VERSION,
+ PARAM_PG_VERSION,
+ PARAM_HOOKS_SETUP_FUNCTION,
+ PARAM_NO_TXINFO,
+ PARAM_RELMETA_CACHE_SIZE
+} OutputPluginParamKey;
+
+typedef struct {
+ const char * const paramname;
+ int paramkey;
+} OutputPluginParam;
+
+/* Oh, if only C had switch on strings */
+static OutputPluginParam param_lookup[] = {
+ {"max_proto_version", PARAM_MAX_PROTOCOL_VERSION},
+ {"min_proto_version", PARAM_MIN_PROTOCOL_VERSION},
+ {"proto_format", PARAM_PROTOCOL_FORMAT},
+ {"expected_encoding", PARAM_EXPECTED_ENCODING},
+ {"binary.bigendian", PARAM_BINARY_BIGENDIAN},
+ {"binary.sizeof_datum", PARAM_BINARY_SIZEOF_DATUM},
+ {"binary.sizeof_int", PARAM_BINARY_SIZEOF_INT},
+ {"binary.sizeof_long", PARAM_BINARY_SIZEOF_LONG},
+ {"binary.float4_byval", PARAM_BINARY_FLOAT4BYVAL},
+ {"binary.float8_byval", PARAM_BINARY_FLOAT8BYVAL},
+ {"binary.integer_datetimes", PARAM_BINARY_INTEGER_DATETIMES},
+ {"binary.want_internal_basetypes", PARAM_BINARY_WANT_INTERNAL_BASETYPES},
+ {"binary.want_binary_basetypes", PARAM_BINARY_WANT_BINARY_BASETYPES},
+ {"binary.basetypes_major_version", PARAM_BINARY_BASETYPES_MAJOR_VERSION},
+ {"pg_version", PARAM_PG_VERSION},
+ {"hooks.setup_function", PARAM_HOOKS_SETUP_FUNCTION},
+ {"no_txinfo", PARAM_NO_TXINFO},
+ {"relmeta_cache_size", PARAM_RELMETA_CACHE_SIZE},
+ {NULL, PARAM_UNRECOGNISED}
+};
+
+/*
+ * Look up a param name to find the enum value for the
+ * param, or PARAM_UNRECOGNISED if not found.
+ */
+static int
+get_param_key(const char * const param_name)
+{
+ OutputPluginParam *param = ¶m_lookup[0];
+
+ do {
+ if (strcmp(param->paramname, param_name) == 0)
+ return param->paramkey;
+ param++;
+ } while (param->paramname != NULL);
+
+ return PARAM_UNRECOGNISED;
+}
+
+
+void
+process_parameters_v1(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ ListCell *lc;
+
+ /*
+ * max_proto_version and min_proto_version are specified
+ * as required, and must be parsed before anything else.
+ *
+ * TODO: We should still parse them as optional and
+ * delay the ERROR until after the startup reply.
+ */
+ val = get_param(options, "max_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_max_proto_version = DatumGetUInt32(val);
+
+ val = get_param(options, "min_proto_version", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+ data->client_min_proto_version = DatumGetUInt32(val);
+
+ /* Examine all the other params in the v1 message. */
+ foreach(lc, options)
+ {
+ DefElem *elem = lfirst(lc);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Check each param, whether or not we recognise it */
+ switch(get_param_key(elem->defname))
+ {
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+
+ case PARAM_BINARY_BIGENDIAN:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_bigendian_set = true;
+ data->client_binary_bigendian = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_DATUM:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofdatum = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_INT:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeofint = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_SIZEOF_LONG:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_sizeoflong = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_FLOAT4BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_FLOAT8BYVAL:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_float4byval_set = true;
+ data->client_binary_float4byval = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_INTEGER_DATETIMES:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_binary_intdatetimes_set = true;
+ data->client_binary_intdatetimes = DatumGetBool(val);
+ break;
+
+ case PARAM_PROTOCOL_FORMAT:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_STRING);
+ data->client_protocol_format = DatumGetCString(val);
+ break;
+
+ case PARAM_EXPECTED_ENCODING:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_STRING);
+ data->client_expected_encoding = DatumGetCString(val);
+ break;
+
+ case PARAM_PG_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_pg_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_BINARY_WANT_INTERNAL_BASETYPES:
+ /* check if we want to use internal data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_internal_basetypes_set = true;
+ data->client_want_internal_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_WANT_BINARY_BASETYPES:
+ /* check if we want to use binary data representation */
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_want_binary_basetypes_set = true;
+ data->client_want_binary_basetypes = DatumGetBool(val);
+ break;
+
+ case PARAM_BINARY_BASETYPES_MAJOR_VERSION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
+ data->client_binary_basetypes_major_version = DatumGetUInt32(val);
+ break;
+
+ case PARAM_HOOKS_SETUP_FUNCTION:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_QUALIFIED_NAME);
+ data->hooks_setup_funcname = (List*) PointerGetDatum(val);
+ break;
+
+ case PARAM_NO_TXINFO:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_BOOL);
+ data->client_no_txinfo = DatumGetBool(val);
+ break;
+
+ case PARAM_RELMETA_CACHE_SIZE:
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_INT32);
+ data->client_relmeta_cache_size = DatumGetInt32(val);
+ break;
+
+ case PARAM_UNRECOGNISED:
+ ereport(DEBUG1,
+ (errmsg("Unrecognised pglogical parameter %s ignored", elem->defname)));
+ break;
+ }
+ }
+}
+
+/*
+ * Read parameters sent by client at startup and store recognised
+ * ones in the parameters PGLogicalOutputData.
+ *
+ * The PGLogicalOutputData must have all client-supplied parameter fields
+ * zeroed, such as by memset or palloc0, since values not supplied
+ * by the client are not set.
+ */
+int
+process_parameters(List *options, PGLogicalOutputData *data)
+{
+ Datum val;
+ bool found;
+ int params_format;
+
+ val = get_param(options, "startup_params_format", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+
+ params_format = DatumGetUInt32(val);
+
+ if (params_format == 1)
+ {
+ process_parameters_v1(options, data);
+ }
+
+ return params_format;
+}
+
+static Datum
+get_param_value(DefElem *elem, bool null_ok, PGLogicalOutputParamType type)
+{
+ /* Check for NULL value */
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ {
+ if (null_ok)
+ return (Datum) 0;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("parameter \"%s\" cannot be NULL", elem->defname)));
+ }
+
+ switch (type)
+ {
+ case OUTPUT_PARAM_TYPE_UINT32:
+ return UInt32GetDatum(parse_param_uint32(elem));
+ case OUTPUT_PARAM_TYPE_INT32:
+ return Int32GetDatum(parse_param_int32(elem));
+ case OUTPUT_PARAM_TYPE_BOOL:
+ return BoolGetDatum(parse_param_bool(elem));
+ case OUTPUT_PARAM_TYPE_STRING:
+ return PointerGetDatum(pstrdup(strVal(elem->arg)));
+ case OUTPUT_PARAM_TYPE_QUALIFIED_NAME:
+ return PointerGetDatum(textToQualifiedNameList(cstring_to_text(pstrdup(strVal(elem->arg)))));
+ default:
+ elog(ERROR, "unknown parameter type %d", type);
+ }
+}
+
+/*
+ * Param parsing
+ *
+ * This is not exactly fast but since it's only called on replication start
+ * we'll leave it for now.
+ */
+static Datum
+get_param(List *options, const char *name, bool missing_ok, bool null_ok,
+ PGLogicalOutputParamType type, bool *found)
+{
+ ListCell *option;
+
+ *found = false;
+
+ foreach(option, options)
+ {
+ DefElem *elem = lfirst(option);
+
+ Assert(elem->arg == NULL || IsA(elem->arg, String));
+
+ /* Search until matching parameter found */
+ if (pg_strcasecmp(name, elem->defname))
+ continue;
+
+ *found = true;
+
+ return get_param_value(elem, null_ok, type);
+ }
+
+ if (!missing_ok)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("missing required parameter \"%s\"", name)));
+
+ return (Datum) 0;
+}
+
+static bool
+parse_param_bool(DefElem *elem)
+{
+ bool res;
+
+ if (!parse_bool(strVal(elem->arg), &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse boolean value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return res;
+}
+
+static uint32
+parse_param_uint32(DefElem *elem)
+{
+ int64 res;
+
+ if (!scanint8(strVal(elem->arg), true, &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse integer value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ if (res > PG_UINT32_MAX || res < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("value \"%s\" out of range for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return (uint32) res;
+}
+
+static int32
+parse_param_int32(DefElem *elem)
+{
+ int64 res;
+
+ if (!scanint8(strVal(elem->arg), true, &res))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("could not parse integer value \"%s\" for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ if (res > PG_INT32_MAX || res < PG_INT32_MIN)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("value \"%s\" out of range for parameter \"%s\"",
+ strVal(elem->arg), elem->defname)));
+
+ return (int32) res;
+}
+
+static List*
+add_startup_msg_s(List *l, char *key, char *val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(val)));
+}
+
+static List*
+add_startup_msg_i(List *l, char *key, int val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(psprintf("%d", val))));
+}
+
+static List*
+add_startup_msg_b(List *l, char *key, bool val)
+{
+ return lappend(l, makeDefElem(key, (Node*)makeString(val ? "t" : "f")));
+}
+
+/*
+ * This builds the protocol startup message, which is always the first
+ * message on the wire after the client sends START_REPLICATION.
+ *
+ * It confirms to the client that we could apply requested options, and
+ * tells the client our capabilities.
+ *
+ * Any additional parameters provided by the startup hook are also output
+ * now.
+ *
+ * The output param 'msg' is a null-terminated char* palloc'd in the current
+ * memory context and the length 'len' of that string that is valid. The caller
+ * should pfree the result after use.
+ *
+ * This is a bit less efficient than direct pq_sendblah calls, but
+ * separates config handling from the protocol implementation, and
+ * it's not like startup msg performance matters much.
+ */
+List *
+prepare_startup_message(PGLogicalOutputData *data)
+{
+ ListCell *lc;
+ List *l = NIL;
+
+ l = add_startup_msg_s(l, "max_proto_version", "1");
+ l = add_startup_msg_s(l, "min_proto_version", "1");
+
+ /* We don't support understand column types yet */
+ l = add_startup_msg_b(l, "coltypes", false);
+
+ /* Info about our Pg host */
+ l = add_startup_msg_i(l, "pg_version_num", PG_VERSION_NUM);
+ l = add_startup_msg_s(l, "pg_version", PG_VERSION);
+ l = add_startup_msg_i(l, "pg_catversion", CATALOG_VERSION_NO);
+
+ l = add_startup_msg_s(l, "database_encoding", (char*)GetDatabaseEncodingName());
+
+ l = add_startup_msg_s(l, "encoding", (char*)pg_encoding_to_char(data->field_datum_encoding));
+
+ l = add_startup_msg_b(l, "forward_changeset_origins",
+ data->forward_changeset_origins);
+
+ /* and ourselves */
+ l = add_startup_msg_s(l, "pglogical_output_version",
+ PGLOGICAL_OUTPUT_VERSION);
+ l = add_startup_msg_i(l, "pglogical_output_version",
+ PGLOGICAL_OUTPUT_VERSION_NUM);
+
+ /* binary options enabled */
+ l = add_startup_msg_b(l, "binary.internal_basetypes",
+ data->allow_internal_basetypes);
+ l = add_startup_msg_b(l, "binary.binary_basetypes",
+ data->allow_binary_basetypes);
+
+ /* Binary format characteristics of server */
+ l = add_startup_msg_i(l, "binary.basetypes_major_version", PG_VERSION_NUM/100);
+ l = add_startup_msg_i(l, "binary.sizeof_int", sizeof(int));
+ l = add_startup_msg_i(l, "binary.sizeof_long", sizeof(long));
+ l = add_startup_msg_i(l, "binary.sizeof_datum", sizeof(Datum));
+ l = add_startup_msg_i(l, "binary.maxalign", MAXIMUM_ALIGNOF);
+ l = add_startup_msg_b(l, "binary.bigendian", server_bigendian());
+ l = add_startup_msg_b(l, "binary.float4_byval", server_float4_byval());
+ l = add_startup_msg_b(l, "binary.float8_byval", server_float8_byval());
+ l = add_startup_msg_b(l, "binary.integer_datetimes", server_integer_datetimes());
+ /* We don't know how to send in anything except our host's format */
+ l = add_startup_msg_i(l, "binary.binary_pg_version",
+ PG_VERSION_NUM/100);
+
+ l = add_startup_msg_b(l, "no_txinfo", data->client_no_txinfo);
+
+
+ /*
+ * Confirm that we've enabled any requested hook functions.
+ */
+ l = add_startup_msg_b(l, "hooks.startup_hook_enabled",
+ data->hooks.startup_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.shutdown_hook_enabled",
+ data->hooks.shutdown_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.row_filter_enabled",
+ data->hooks.row_filter_hook != NULL);
+ l = add_startup_msg_b(l, "hooks.transaction_filter_enabled",
+ data->hooks.txn_filter_hook != NULL);
+
+ /* Cache control and other misc options */
+ l = add_startup_msg_i(l, "relmeta_cache_size",
+ data->relmeta_cache_size);
+
+
+ /*
+ * Output any extra params supplied by a startup hook by appending
+ * them verbatim to the params list.
+ */
+ foreach(lc, data->extra_startup_params)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ l = lappend(l, param);
+ }
+
+ return l;
+}
diff --git a/contrib/pglogical_output/pglogical_config.h b/contrib/pglogical_output/pglogical_config.h
new file mode 100644
index 0000000..8f620bd
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_config.h
@@ -0,0 +1,55 @@
+#ifndef PG_LOGICAL_CONFIG_H
+#define PG_LOGICAL_CONFIG_H
+
+#ifndef PG_VERSION_NUM
+#error <postgres.h> must be included first
+#endif
+
+#include "nodes/pg_list.h"
+#include "pglogical_output.h"
+
+inline static bool
+server_float4_byval(void)
+{
+#ifdef USE_FLOAT4_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_float8_byval(void)
+{
+#ifdef USE_FLOAT8_BYVAL
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_integer_datetimes(void)
+{
+#ifdef USE_INTEGER_DATETIMES
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline static bool
+server_bigendian(void)
+{
+#ifdef WORDS_BIGENDIAN
+ return true;
+#else
+ return false;
+#endif
+}
+
+extern int process_parameters(List *options, PGLogicalOutputData *data);
+
+extern List * prepare_startup_message(PGLogicalOutputData *data);
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_hooks.c b/contrib/pglogical_output/pglogical_hooks.c
new file mode 100644
index 0000000..47aa9ba
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.c
@@ -0,0 +1,235 @@
+#include "postgres.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#ifdef HAVE_REPLICATION_ORIGINS
+#include "replication/origin.h"
+#endif
+
+#include "parser/parse_func.h"
+
+#include "utils/acl.h"
+#include "utils/lsyscache.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_hooks.h"
+#include "pglogical_output.h"
+
+/*
+ * Returns Oid of the hooks function specified in funcname.
+ *
+ * Error is thrown if function doesn't exist or doen't return correct datatype
+ * or is volatile.
+ */
+static Oid
+get_hooks_function_oid(List *funcname)
+{
+ Oid funcid;
+ Oid funcargtypes[1];
+
+ funcargtypes[0] = INTERNALOID;
+
+ /* find the the function */
+ funcid = LookupFuncName(funcname, 1, funcargtypes, false);
+
+ /* Validate that the function returns void */
+ if (get_func_rettype(funcid) != VOIDOID)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must return void",
+ NameListToString(funcname))));
+ }
+
+ if (func_volatile(funcid) == PROVOLATILE_VOLATILE)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s must not be VOLATILE",
+ NameListToString(funcname))));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(funcname))));
+ }
+
+ return funcid;
+}
+
+/*
+ * If a hook setup function was specified in the startup parameters, look it up
+ * in the catalogs, check permissions, call it, and store the resulting hook
+ * info struct.
+ */
+void
+load_hooks(PGLogicalOutputData *data)
+{
+ Oid hooks_func;
+ MemoryContext old_ctxt;
+ bool txn_started = false;
+
+ if (!IsTransactionState())
+ {
+ txn_started = true;
+ StartTransactionCommand();
+ }
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+ hooks_func = get_hooks_function_oid(data->hooks_setup_funcname);
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) OidFunctionCall1(hooks_func, PointerGetDatum(&data->hooks));
+ MemoryContextSwitchTo(old_ctxt);
+
+ elog(DEBUG3, "pglogical_output: Loaded hooks from function %u. Hooks are: \n"
+ "\tstartup_hook: %p\n"
+ "\tshutdown_hook: %p\n"
+ "\trow_filter_hook: %p\n"
+ "\ttxn_filter_hook: %p\n"
+ "\thooks_private_data: %p\n",
+ hooks_func,
+ data->hooks.startup_hook,
+ data->hooks.shutdown_hook,
+ data->hooks.row_filter_hook,
+ data->hooks.txn_filter_hook,
+ data->hooks.hooks_private_data);
+ }
+
+ if (txn_started)
+ CommitTransactionCommand();
+}
+
+void
+call_startup_hook(PGLogicalOutputData *data, List *plugin_params)
+{
+ struct PGLogicalStartupHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.startup_hook != NULL)
+ {
+ bool tx_started = false;
+
+ args.private_data = data->hooks.hooks_private_data;
+ args.in_params = plugin_params;
+ args.out_params = NIL;
+
+ elog(DEBUG3, "calling pglogical startup hook");
+
+ if (!IsTransactionState())
+ {
+ tx_started = true;
+ StartTransactionCommand();
+ }
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.startup_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ if (tx_started)
+ CommitTransactionCommand();
+
+ data->extra_startup_params = args.out_params;
+ /* The startup hook might change the private data seg */
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical startup hook");
+ }
+}
+
+void
+call_shutdown_hook(PGLogicalOutputData *data)
+{
+ struct PGLogicalShutdownHookArgs args;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.shutdown_hook != NULL)
+ {
+ args.private_data = data->hooks.hooks_private_data;
+
+ elog(DEBUG3, "calling pglogical shutdown hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ (void) (*data->hooks.shutdown_hook)(&args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ data->hooks.hooks_private_data = args.private_data;
+
+ elog(DEBUG3, "called pglogical shutdown hook");
+ }
+}
+
+/*
+ * Decide if the individual change should be filtered out by
+ * calling a client-provided hook.
+ */
+bool
+call_row_filter_hook(PGLogicalOutputData *data, ReorderBufferTXN *txn,
+ Relation rel, ReorderBufferChange *change)
+{
+ struct PGLogicalRowFilterArgs hook_args;
+ MemoryContext old_ctxt;
+ bool ret = true;
+
+ if (data->hooks.row_filter_hook != NULL)
+ {
+ hook_args.change_type = change->action;
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.changed_rel = rel;
+ hook_args.change = change;
+
+ elog(DEBUG3, "calling pglogical row filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.row_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical row filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
+
+bool
+call_txn_filter_hook(PGLogicalOutputData *data, RepOriginId txn_origin)
+{
+ struct PGLogicalTxnFilterArgs hook_args;
+ bool ret = true;
+ MemoryContext old_ctxt;
+
+ if (data->hooks.txn_filter_hook != NULL)
+ {
+ hook_args.private_data = data->hooks.hooks_private_data;
+ hook_args.origin_id = txn_origin;
+
+ elog(DEBUG3, "calling pglogical txn filter hook");
+
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt);
+ ret = (*data->hooks.txn_filter_hook)(&hook_args);
+ MemoryContextSwitchTo(old_ctxt);
+
+ /* Filter hooks shouldn't change the private data ptr */
+ Assert(data->hooks.hooks_private_data == hook_args.private_data);
+
+ elog(DEBUG3, "called pglogical txn filter hook, returned %d", (int)ret);
+ }
+
+ return ret;
+}
diff --git a/contrib/pglogical_output/pglogical_hooks.h b/contrib/pglogical_output/pglogical_hooks.h
new file mode 100644
index 0000000..5dd9705
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_hooks.h
@@ -0,0 +1,23 @@
+#ifndef PGLOGICAL_HOOKS_H
+#define PGLOGICAL_HOOKS_H
+
+#include "replication/reorderbuffer.h"
+
+/* public interface for hooks */
+#include "pglogical_output/hooks.h"
+#include "pglogical_output.h"
+
+extern void load_hooks(PGLogicalOutputData *data);
+
+extern void call_startup_hook(PGLogicalOutputData *data, List *plugin_params);
+
+extern void call_shutdown_hook(PGLogicalOutputData *data);
+
+extern bool call_row_filter_hook(PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, Relation rel, ReorderBufferChange *change);
+
+extern bool call_txn_filter_hook(PGLogicalOutputData *data,
+ RepOriginId txn_origin);
+
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_infofuncs.c b/contrib/pglogical_output/pglogical_infofuncs.c
new file mode 100644
index 0000000..055cd9b
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_infofuncs.c
@@ -0,0 +1,55 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_infofuncs.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_infofuncs.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+#include "fmgr.h"
+
+#include "utils/builtins.h"
+
+#include "pglogical_output.h"
+
+Datum pglogical_output_version(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglogical_output_version);
+
+Datum pglogical_output_version_num(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglogical_output_version_num);
+
+Datum pglogical_output_proto_version(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglogical_output_proto_version);
+
+Datum pglogical_output_min_proto_version(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglogical_output_min_proto_version);
+
+Datum
+pglogical_output_version(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_TEXT_P(cstring_to_text(PGLOGICAL_OUTPUT_VERSION));
+}
+
+Datum
+pglogical_output_version_num(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_INT32(PGLOGICAL_OUTPUT_VERSION_NUM);
+}
+
+Datum
+pglogical_output_proto_version(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_INT32(PGLOGICAL_PROTO_VERSION_NUM);
+}
+
+Datum
+pglogical_output_min_proto_version(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_INT32(PGLOGICAL_PROTO_MIN_VERSION_NUM);
+}
diff --git a/contrib/pglogical_output/pglogical_output--1.0.0.sql b/contrib/pglogical_output/pglogical_output--1.0.0.sql
new file mode 100644
index 0000000..d6c330b
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output--1.0.0.sql
@@ -0,0 +1,11 @@
+CREATE FUNCTION pglogical_output_version() RETURNS text
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+CREATE FUNCTION pglogical_output_version_num() RETURNS integer
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+CREATE FUNCTION pglogical_output_proto_version() RETURNS integer
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+CREATE FUNCTION pglogical_output_min_proto_version() RETURNS integer
+LANGUAGE c AS 'MODULE_PATHNAME';
diff --git a/contrib/pglogical_output/pglogical_output.c b/contrib/pglogical_output/pglogical_output.c
new file mode 100644
index 0000000..ca809e7
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.c
@@ -0,0 +1,569 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.c
+ * Logical Replication output plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output/compat.h"
+#include "pglogical_config.h"
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+#include "pglogical_hooks.h"
+#include "pglogical_relmetacache.h"
+
+#include "access/hash.h"
+#include "access/sysattr.h"
+#include "access/xact.h"
+
+#include "catalog/pg_class.h"
+#include "catalog/pg_proc.h"
+#include "catalog/pg_type.h"
+
+#include "mb/pg_wchar.h"
+
+#include "nodes/parsenodes.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/output_plugin.h"
+#include "replication/logical.h"
+#ifdef HAVE_REPLICATION_ORIGINS
+#include "replication/origin.h"
+#endif
+
+#include "utils/builtins.h"
+#include "utils/catcache.h"
+#include "utils/guc.h"
+#include "utils/int8.h"
+#include "utils/inval.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/relcache.h"
+#include "utils/syscache.h"
+#include "utils/typcache.h"
+
+PG_MODULE_MAGIC;
+
+extern void _PG_output_plugin_init(OutputPluginCallbacks *cb);
+
+/* These must be available to pg_dlsym() */
+static void pg_decode_startup(LogicalDecodingContext * ctx,
+ OutputPluginOptions *opt, bool is_init);
+static void pg_decode_shutdown(LogicalDecodingContext * ctx);
+static void pg_decode_begin_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn);
+static void pg_decode_commit_txn(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+static void pg_decode_change(LogicalDecodingContext *ctx,
+ ReorderBufferTXN *txn, Relation rel,
+ ReorderBufferChange *change);
+
+#ifdef HAVE_REPLICATION_ORIGINS
+static bool pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id);
+#endif
+
+static void send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message);
+
+static bool startup_message_sent = false;
+
+/* specify output plugin callbacks */
+void
+_PG_output_plugin_init(OutputPluginCallbacks *cb)
+{
+ AssertVariableIsOfType(&_PG_output_plugin_init, LogicalOutputPluginInit);
+
+ cb->startup_cb = pg_decode_startup;
+ cb->begin_cb = pg_decode_begin_txn;
+ cb->change_cb = pg_decode_change;
+ cb->commit_cb = pg_decode_commit_txn;
+#ifdef HAVE_REPLICATION_ORIGINS
+ cb->filter_by_origin_cb = pg_decode_origin_filter;
+#endif
+ cb->shutdown_cb = pg_decode_shutdown;
+}
+
+static bool
+check_binary_compatibility(PGLogicalOutputData *data)
+{
+ if (data->client_binary_basetypes_major_version != PG_VERSION_NUM / 100)
+ return false;
+
+ if (data->client_binary_bigendian_set
+ && data->client_binary_bigendian != server_bigendian())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian mis-match");
+ return false;
+ }
+
+ if (data->client_binary_sizeofdatum != 0
+ && data->client_binary_sizeofdatum != sizeof(Datum))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(Datum) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeofint != 0
+ && data->client_binary_sizeofint != sizeof(int))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(int) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeoflong != 0
+ && data->client_binary_sizeoflong != sizeof(long))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(long) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float4byval_set
+ && data->client_binary_float4byval != server_float4_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float4byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_float8byval_set
+ && data->client_binary_float8byval != server_float8_byval())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian float8byval mismatch");
+ return false;
+ }
+
+ if (data->client_binary_intdatetimes_set
+ && data->client_binary_intdatetimes != server_integer_datetimes())
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian integer datetimes mismatch");
+ return false;
+ }
+
+ return true;
+}
+
+/* initialize this plugin */
+static void
+pg_decode_startup(LogicalDecodingContext * ctx, OutputPluginOptions *opt,
+ bool is_init)
+{
+ PGLogicalOutputData *data = palloc0(sizeof(PGLogicalOutputData));
+
+ data->context = AllocSetContextCreate(TopMemoryContext,
+ "pglogical conversion context",
+ ALLOCSET_DEFAULT_MINSIZE,
+ ALLOCSET_DEFAULT_INITSIZE,
+ ALLOCSET_DEFAULT_MAXSIZE);
+ data->allow_internal_basetypes = false;
+ data->allow_binary_basetypes = false;
+
+
+ ctx->output_plugin_private = data;
+
+ /*
+ * This is replication start and not slot initialization.
+ *
+ * Parse and validate options passed by the client.
+ */
+ if (!is_init)
+ {
+ int params_format;
+
+ /*
+ * Ideally we'd send the startup message immediately. That way
+ * it'd arrive before any error we emit if we see incompatible
+ * options sent by the client here. That way the client could
+ * possibly adjust its options and reconnect. It'd also make
+ * sure the client gets the startup message in a timely way if
+ * the server is idle, since otherwise it could be a while
+ * before the next callback.
+ *
+ * The decoding plugin API doesn't let us write to the stream
+ * from here, though, so we have to delay the startup message
+ * until the first change processed on the stream, in a begin
+ * callback.
+ *
+ * If we ERROR there, the startup message is buffered but not
+ * sent since the callback didn't finish. So we'd have to send
+ * the startup message, finish the callback and check in the
+ * next callback if we need to ERROR.
+ *
+ * That's a bit much hoop jumping, so for now ERRORs are
+ * immediate. A way to emit a message from the startup callback
+ * is really needed to change that.
+ */
+ startup_message_sent = false;
+
+ /* Now parse the rest of the params and ERROR if we see any we don't recognise */
+ params_format = process_parameters(ctx->output_plugin_options, data);
+
+ if (params_format != 1)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent startup parameters in format %d but we only support format 1",
+ params_format)));
+
+ if (data->client_min_proto_version > PGLOGICAL_PROTO_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent min_proto_version=%d but we only support protocol %d or lower",
+ data->client_min_proto_version, PGLOGICAL_PROTO_VERSION_NUM)));
+
+ if (data->client_max_proto_version < PGLOGICAL_PROTO_MIN_VERSION_NUM)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client sent max_proto_version=%d but we only support protocol %d or higher",
+ data->client_max_proto_version, PGLOGICAL_PROTO_MIN_VERSION_NUM)));
+
+ /*
+ * Set correct protocol format.
+ *
+ * This is the output plugin protocol format, this is different
+ * from the individual fields binary vs textual format.
+ */
+ if (data->client_protocol_format != NULL
+ && strcmp(data->client_protocol_format, "json") == 0)
+ {
+ data->api = pglogical_init_api(PGLogicalProtoJson);
+ opt->output_type = OUTPUT_PLUGIN_TEXTUAL_OUTPUT;
+ }
+ else if ((data->client_protocol_format != NULL
+ && strcmp(data->client_protocol_format, "native") == 0)
+ || data->client_protocol_format == NULL)
+ {
+ data->api = pglogical_init_api(PGLogicalProtoNative);
+ opt->output_type = OUTPUT_PLUGIN_BINARY_OUTPUT;
+
+ if (data->client_no_txinfo)
+ {
+ elog(WARNING, "no_txinfo option ignored for protocols other than json");
+ data->client_no_txinfo = false;
+ }
+ }
+ else
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("client requested protocol %s but only \"json\" or \"native\" are supported",
+ data->client_protocol_format)));
+ }
+
+ /* check for encoding match if specific encoding demanded by client */
+ if (data->client_expected_encoding != NULL
+ && strlen(data->client_expected_encoding) != 0)
+ {
+ int wanted_encoding = pg_char_to_encoding(data->client_expected_encoding);
+
+ if (wanted_encoding == -1)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("unrecognised encoding name %s passed to expected_encoding",
+ data->client_expected_encoding)));
+
+ if (opt->output_type == OUTPUT_PLUGIN_TEXTUAL_OUTPUT)
+ {
+ /*
+ * datum encoding must match assigned client_encoding in text
+ * proto, since everything is subject to client_encoding
+ * conversion.
+ */
+ if (wanted_encoding != pg_get_client_encoding())
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("expected_encoding must be unset or match client_encoding in text protocols")));
+ }
+ else
+ {
+ /*
+ * currently in the binary protocol we can only emit encoded
+ * datums in the server encoding. There's no support for encoding
+ * conversion.
+ */
+ if (wanted_encoding != GetDatabaseEncoding())
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("encoding conversion for binary datum not supported yet"),
+ errdetail("expected_encoding %s must be unset or match server_encoding %s",
+ data->client_expected_encoding, GetDatabaseEncodingName())));
+ }
+
+ data->field_datum_encoding = wanted_encoding;
+ }
+
+ /*
+ * It's obviously not possible to send binary representatio of data
+ * unless we use the binary output.
+ */
+ if (opt->output_type == OUTPUT_PLUGIN_BINARY_OUTPUT &&
+ data->client_want_internal_basetypes)
+ {
+ data->allow_internal_basetypes =
+ check_binary_compatibility(data);
+ }
+
+ if (opt->output_type == OUTPUT_PLUGIN_BINARY_OUTPUT &&
+ data->client_want_binary_basetypes &&
+ data->client_binary_basetypes_major_version == PG_VERSION_NUM / 100)
+ {
+ data->allow_binary_basetypes = true;
+ }
+
+ /*
+ * 9.4 lacks origins info so don't forward it.
+ *
+ * There's currently no knob for clients to use to suppress
+ * this info and it's sent if it's supported and available.
+ */
+ if (PG_VERSION_NUM/100 == 904)
+ data->forward_changeset_origins = false;
+ else
+ data->forward_changeset_origins = true;
+
+ if (data->hooks_setup_funcname != NIL)
+ {
+
+ data->hooks_mctxt = AllocSetContextCreate(ctx->context,
+ "pglogical_output hooks context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+ load_hooks(data);
+ call_startup_hook(data, ctx->output_plugin_options);
+ }
+
+ if (data->client_relmeta_cache_size < -1)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("relmeta_cache_size must be -1, 0, or positive")));
+ }
+
+ /*
+ * Relation metadata cache configuration.
+ *
+ * TODO: support fixed size cache
+ *
+ * Need a LRU for eviction, and need to implement a new message type for
+ * cache purge notifications for clients. In the mean time force it to 0
+ * (off). The client will be told via a startup param and must respect
+ * that.
+ */
+ if (data->client_relmeta_cache_size != 0
+ && data->client_relmeta_cache_size != -1)
+ {
+ ereport(INFO,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("fixed size cache not supported, forced to off"),
+ errdetail("only relmeta_cache_size=0 (off) or relmeta_cache_size=-1 (unlimited) supported")));
+
+ data->relmeta_cache_size = 0;
+ }
+ else
+ {
+ /* ack client request */
+ data->relmeta_cache_size = data->client_relmeta_cache_size;
+ }
+
+ /* if cache enabled, init it */
+ if (data->relmeta_cache_size != 0)
+ pglogical_init_relmetacache();
+ }
+}
+
+/*
+ * BEGIN callback
+ */
+void
+pg_decode_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+ bool send_replication_origin = data->forward_changeset_origins;
+
+ if (!startup_message_sent)
+ send_startup_message(ctx, data, false /* can't be last message */);
+
+#ifdef HAVE_REPLICATION_ORIGINS
+ /* If the record didn't originate locally, send origin info */
+ send_replication_origin &= txn->origin_id != InvalidRepOriginId;
+#endif
+
+ OutputPluginPrepareWrite(ctx, !send_replication_origin);
+ data->api->write_begin(ctx->out, data, txn);
+
+#ifdef HAVE_REPLICATION_ORIGINS
+ if (send_replication_origin)
+ {
+ char *origin;
+
+ /* Message boundary */
+ OutputPluginWrite(ctx, false);
+ OutputPluginPrepareWrite(ctx, true);
+
+ /*
+ * XXX: which behaviour we want here?
+ *
+ * Alternatives:
+ * - don't send origin message if origin name not found
+ * (that's what we do now)
+ * - throw error - that will break replication, not good
+ * - send some special "unknown" origin
+ */
+ if (data->api->write_origin &&
+ replorigin_by_oid(txn->origin_id, true, &origin))
+ data->api->write_origin(ctx->out, origin, txn->origin_lsn);
+ }
+#endif
+
+ OutputPluginWrite(ctx, true);
+}
+
+/*
+ * COMMIT callback
+ */
+void
+pg_decode_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_commit(ctx->out, data, txn, commit_lsn);
+ OutputPluginWrite(ctx, true);
+}
+
+void
+pg_decode_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
+ Relation relation, ReorderBufferChange *change)
+{
+ PGLogicalOutputData *data = ctx->output_plugin_private;
+ MemoryContext old;
+ struct PGLRelMetaCacheEntry *cached_relmeta = NULL;
+
+
+ /* First check the table filter */
+ if (!call_row_filter_hook(data, txn, relation, change))
+ return;
+
+ /* Avoid leaking memory by using and resetting our own context */
+ old = MemoryContextSwitchTo(data->context);
+
+ /*
+ * If the protocol wants to write relation information and the client
+ * isn't known to have metadata cached for this relation already,
+ * send relation metadata.
+ *
+ * TODO: track hit/miss stats
+ */
+ if (data->api->write_rel != NULL &&
+ !pglogical_cache_relmeta(data, relation, &cached_relmeta))
+ {
+ OutputPluginPrepareWrite(ctx, false);
+ data->api->write_rel(ctx->out, data, relation, cached_relmeta);
+ OutputPluginWrite(ctx, false);
+ }
+
+ /* Send the data */
+ switch (change->action)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_insert(ctx->out, data, relation,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ {
+ HeapTuple oldtuple = change->data.tp.oldtuple ?
+ &change->data.tp.oldtuple->tuple : NULL;
+
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_update(ctx->out, data, relation, oldtuple,
+ &change->data.tp.newtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ break;
+ }
+ case REORDER_BUFFER_CHANGE_DELETE:
+ if (change->data.tp.oldtuple)
+ {
+ OutputPluginPrepareWrite(ctx, true);
+ data->api->write_delete(ctx->out, data, relation,
+ &change->data.tp.oldtuple->tuple);
+ OutputPluginWrite(ctx, true);
+ }
+ else
+ elog(DEBUG1, "didn't send DELETE change because of missing oldtuple");
+ break;
+ default:
+ Assert(false);
+ }
+
+ /* Cleanup */
+ MemoryContextSwitchTo(old);
+ MemoryContextReset(data->context);
+}
+
+#ifdef HAVE_REPLICATION_ORIGINS
+/*
+ * Decide if the whole transaction with specific origin should be filtered out.
+ */
+static bool
+pg_decode_origin_filter(LogicalDecodingContext *ctx,
+ RepOriginId origin_id)
+{
+ PGLogicalOutputData *data = ctx->output_plugin_private;
+
+ if (!call_txn_filter_hook(data, origin_id))
+ return true;
+
+ return false;
+}
+#endif
+
+static void
+send_startup_message(LogicalDecodingContext *ctx,
+ PGLogicalOutputData *data, bool last_message)
+{
+ List *msg;
+
+ Assert(!startup_message_sent);
+
+ msg = prepare_startup_message(data);
+
+ /*
+ * We could free the extra_startup_params DefElem list here, but it's
+ * pretty harmless to just ignore it, since it's in the decoding memory
+ * context anyway, and we don't know if it's safe to free the defnames or
+ * not.
+ */
+
+ OutputPluginPrepareWrite(ctx, last_message);
+ data->api->write_startup_message(ctx->out, msg);
+ OutputPluginWrite(ctx, last_message);
+
+ pfree(msg);
+
+ startup_message_sent = true;
+}
+
+static void pg_decode_shutdown(LogicalDecodingContext * ctx)
+{
+ PGLogicalOutputData* data = (PGLogicalOutputData*)ctx->output_plugin_private;
+
+ call_shutdown_hook(data);
+
+ if (data->hooks_mctxt != NULL)
+ {
+ MemoryContextDelete(data->hooks_mctxt);
+ data->hooks_mctxt = NULL;
+ }
+}
diff --git a/contrib/pglogical_output/pglogical_output.control.in b/contrib/pglogical_output/pglogical_output.control.in
new file mode 100644
index 0000000..1b86595
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.control.in
@@ -0,0 +1,4 @@
+default_version = '__PGLOGICAL_OUTPUT_VERSION__'
+comment = 'general purpose logical decoding plugin'
+module_pathname = 'pglogical_output'
+superuser = false
diff --git a/contrib/pglogical_output/pglogical_output.h b/contrib/pglogical_output/pglogical_output.h
new file mode 100644
index 0000000..a6bf281
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output.h
@@ -0,0 +1,111 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_output.h
+ * pglogical output plugin
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_output.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_OUTPUT_H
+#define PG_LOGICAL_OUTPUT_H
+
+#include "nodes/parsenodes.h"
+
+#include "replication/logical.h"
+#include "replication/output_plugin.h"
+
+#include "storage/lock.h"
+
+#include "pglogical_output/hooks.h"
+
+#include "pglogical_proto.h"
+
+/* XXYYZZ format version number and human readable version */
+#define PGLOGICAL_OUTPUT_VERSION_NUM 10000
+#define PGLOGICAL_OUTPUT_VERSION "1.0.0"
+
+/* Protocol capabilities */
+#define PGLOGICAL_PROTO_VERSION_NUM 1
+#define PGLOGICAL_PROTO_MIN_VERSION_NUM 1
+
+/*
+ * The name of a hook function. This is used instead of the usual List*
+ * because can serve as a hash key.
+ *
+ * Must be zeroed on allocation if used as a hash key since padding is
+ * *not* ignored on compare.
+ */
+typedef struct HookFuncName
+{
+ /* funcname is more likely to be unique, so goes first */
+ char function[NAMEDATALEN];
+ char schema[NAMEDATALEN];
+} HookFuncName;
+
+struct PGLogicalProtoAPI;
+
+typedef struct PGLogicalOutputData
+{
+ MemoryContext context;
+
+ struct PGLogicalProtoAPI *api;
+
+ /* protocol */
+ bool allow_internal_basetypes;
+ bool allow_binary_basetypes;
+ bool forward_changeset_origins;
+ int field_datum_encoding;
+ int relmeta_cache_size;
+
+ /*
+ * client info
+ *
+ * Lots of this should move to a separate shorter-lived struct used only
+ * during parameter reading, since it contains what the client asked for.
+ * Once we've processed this during startup we don't refer to it again.
+ */
+ uint32 client_pg_version;
+ uint32 client_max_proto_version;
+ uint32 client_min_proto_version;
+ const char *client_expected_encoding;
+ const char *client_protocol_format;
+ uint32 client_binary_basetypes_major_version;
+ bool client_want_internal_basetypes_set;
+ bool client_want_internal_basetypes;
+ bool client_want_binary_basetypes_set;
+ bool client_want_binary_basetypes;
+ bool client_binary_bigendian_set;
+ bool client_binary_bigendian;
+ uint32 client_binary_sizeofdatum;
+ uint32 client_binary_sizeofint;
+ uint32 client_binary_sizeoflong;
+ bool client_binary_float4byval_set;
+ bool client_binary_float4byval;
+ bool client_binary_float8byval_set;
+ bool client_binary_float8byval;
+ bool client_binary_intdatetimes_set;
+ bool client_binary_intdatetimes;
+ bool client_no_txinfo;
+ int client_relmeta_cache_size;
+
+ /* hooks */
+ List *hooks_setup_funcname;
+ struct PGLogicalHooks hooks;
+ MemoryContext hooks_mctxt;
+
+ /* DefElem<String> list populated by startup hook */
+ List *extra_startup_params;
+} PGLogicalOutputData;
+
+typedef struct PGLogicalTupleData
+{
+ Datum values[MaxTupleAttributeNumber];
+ bool nulls[MaxTupleAttributeNumber];
+ bool changed[MaxTupleAttributeNumber];
+} PGLogicalTupleData;
+
+#endif /* PG_LOGICAL_OUTPUT_H */
diff --git a/contrib/pglogical_output/pglogical_output/README b/contrib/pglogical_output/pglogical_output/README
new file mode 100644
index 0000000..5480e5c
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/README
@@ -0,0 +1,7 @@
+/*
+ * This directory contains the public header files for the pglogical_output
+ * extension. It is installed into the PostgreSQL source tree when the extension
+ * is installed.
+ *
+ * These headers are not part of the PostgreSQL project its self.
+ */
diff --git a/contrib/pglogical_output/pglogical_output/compat.h b/contrib/pglogical_output/pglogical_output/compat.h
new file mode 100644
index 0000000..b0b14fc
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/compat.h
@@ -0,0 +1,28 @@
+#ifndef PG_LOGICAL_COMPAT_H
+#define PG_LOGICAL_COMPAT_H
+
+#include "pg_config.h"
+
+/* 9.4 lacks replication origins */
+#if PG_VERSION_NUM >= 90500
+#define HAVE_REPLICATION_ORIGINS
+#else
+/* To allow the same signature on hooks in 9.4 */
+typedef uint16 RepOriginId;
+#define InvalidRepOriginId 0
+#endif
+
+/* 9.4 lacks PG_UINT32_MAX */
+#ifndef PG_UINT32_MAX
+#define PG_UINT32_MAX UINT32_MAX
+#endif
+
+#ifndef PG_INT32_MAX
+#define PG_INT32_MAX INT32_MAX
+#endif
+
+#ifndef PG_INT32_MIN
+#define PG_INT32_MIN INT32_MIN
+#endif
+
+#endif
diff --git a/contrib/pglogical_output/pglogical_output/hooks.h b/contrib/pglogical_output/pglogical_output/hooks.h
new file mode 100644
index 0000000..8766dd7
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_output/hooks.h
@@ -0,0 +1,73 @@
+#ifndef PGLOGICAL_OUTPUT_HOOKS_H
+#define PGLOGICAL_OUTPUT_HOOKS_H
+
+#include "access/xlogdefs.h"
+#include "nodes/pg_list.h"
+#include "utils/rel.h"
+#include "utils/palloc.h"
+#include "replication/reorderbuffer.h"
+
+#include "pglogical_output/compat.h"
+
+/*
+ * This header is to be included by extensions that implement pglogical output
+ * plugin callback hooks for transaction origin and row filtering, etc. It is
+ * installed as "pglogical_output/hooks.h"
+ *
+ * See the README.md and the example in examples/hooks/ for details on hooks.
+ */
+
+
+struct PGLogicalStartupHookArgs
+{
+ void *private_data;
+ List *in_params;
+ List *out_params;
+};
+
+typedef void (*pglogical_startup_hook_fn)(struct PGLogicalStartupHookArgs *args);
+
+
+struct PGLogicalTxnFilterArgs
+{
+ void *private_data;
+ RepOriginId origin_id;
+};
+
+typedef bool (*pglogical_txn_filter_hook_fn)(struct PGLogicalTxnFilterArgs *args);
+
+
+struct PGLogicalRowFilterArgs
+{
+ void *private_data;
+ Relation changed_rel;
+ enum ReorderBufferChangeType change_type;
+ /* detailed row change event from logical decoding */
+ ReorderBufferChange* change;
+};
+
+typedef bool (*pglogical_row_filter_hook_fn)(struct PGLogicalRowFilterArgs *args);
+
+
+struct PGLogicalShutdownHookArgs
+{
+ void *private_data;
+};
+
+typedef void (*pglogical_shutdown_hook_fn)(struct PGLogicalShutdownHookArgs *args);
+
+/*
+ * This struct is passed to the pglogical_get_hooks_fn as the first argument,
+ * typed 'internal', and is unwrapped with `DatumGetPointer`.
+ */
+struct PGLogicalHooks
+{
+ pglogical_startup_hook_fn startup_hook;
+ pglogical_shutdown_hook_fn shutdown_hook;
+ pglogical_txn_filter_hook_fn txn_filter_hook;
+ pglogical_row_filter_hook_fn row_filter_hook;
+ void *hooks_private_data;
+};
+
+
+#endif /* PGLOGICAL_OUTPUT_HOOKS_H */
diff --git a/contrib/pglogical_output/pglogical_proto.c b/contrib/pglogical_output/pglogical_proto.c
new file mode 100644
index 0000000..47a883f
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.c
@@ -0,0 +1,49 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.c
+ * pglogical protocol functions
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto.h"
+#include "pglogical_proto_native.h"
+#include "pglogical_proto_json.h"
+
+PGLogicalProtoAPI *
+pglogical_init_api(PGLogicalProtoType typ)
+{
+ PGLogicalProtoAPI *res = palloc0(sizeof(PGLogicalProtoAPI));
+
+ if (typ == PGLogicalProtoJson)
+ {
+ res->write_rel = NULL;
+ res->write_begin = pglogical_json_write_begin;
+ res->write_commit = pglogical_json_write_commit;
+ res->write_origin = NULL;
+ res->write_insert = pglogical_json_write_insert;
+ res->write_update = pglogical_json_write_update;
+ res->write_delete = pglogical_json_write_delete;
+ res->write_startup_message = json_write_startup_message;
+ }
+ else
+ {
+ res->write_rel = pglogical_write_rel;
+ res->write_begin = pglogical_write_begin;
+ res->write_commit = pglogical_write_commit;
+ res->write_origin = pglogical_write_origin;
+ res->write_insert = pglogical_write_insert;
+ res->write_update = pglogical_write_update;
+ res->write_delete = pglogical_write_delete;
+ res->write_startup_message = write_startup_message;
+ }
+
+ return res;
+}
diff --git a/contrib/pglogical_output/pglogical_proto.h b/contrib/pglogical_output/pglogical_proto.h
new file mode 100644
index 0000000..27897e2
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto.h
@@ -0,0 +1,61 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto.h
+ * pglogical protocol
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_H
+#define PG_LOGICAL_PROTO_H
+
+struct PGLogicalOutputData;
+struct PGLRelMetaCacheEntry;
+
+typedef void (*pglogical_write_rel_fn)(StringInfo out, struct PGLogicalOutputData *data,
+ Relation rel, struct PGLRelMetaCacheEntry *cache_entry);
+
+typedef void (*pglogical_write_begin_fn)(StringInfo out, struct PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+typedef void (*pglogical_write_commit_fn)(StringInfo out, struct PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+typedef void (*pglogical_write_origin_fn)(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn);
+
+typedef void (*pglogical_write_insert_fn)(StringInfo out, struct PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+typedef void (*pglogical_write_update_fn)(StringInfo out, struct PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+typedef void (*pglogical_write_delete_fn)(StringInfo out, struct PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+typedef void (*write_startup_message_fn)(StringInfo out, List *msg);
+
+typedef struct PGLogicalProtoAPI
+{
+ pglogical_write_rel_fn write_rel;
+ pglogical_write_begin_fn write_begin;
+ pglogical_write_commit_fn write_commit;
+ pglogical_write_origin_fn write_origin;
+ pglogical_write_insert_fn write_insert;
+ pglogical_write_update_fn write_update;
+ pglogical_write_delete_fn write_delete;
+ write_startup_message_fn write_startup_message;
+} PGLogicalProtoAPI;
+
+
+typedef enum PGLogicalProtoType
+{
+ PGLogicalProtoNative,
+ PGLogicalProtoJson
+} PGLogicalProtoType;
+
+extern PGLogicalProtoAPI *pglogical_init_api(PGLogicalProtoType typ);
+
+#endif /* PG_LOGICAL_PROTO_H */
diff --git a/contrib/pglogical_output/pglogical_proto_json.c b/contrib/pglogical_output/pglogical_proto_json.c
new file mode 100644
index 0000000..ae5a591
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_json.c
@@ -0,0 +1,204 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_json.c
+ * pglogical protocol functions for json support
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_json.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_output.h"
+#include "pglogical_proto_json.h"
+
+#include "access/sysattr.h"
+#include "access/tuptoaster.h"
+#include "access/xact.h"
+
+#include "catalog/catversion.h"
+#include "catalog/index.h"
+
+#include "catalog/namespace.h"
+#include "catalog/pg_class.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_namespace.h"
+#include "catalog/pg_type.h"
+
+#include "commands/dbcommands.h"
+
+#include "executor/spi.h"
+
+#include "libpq/pqformat.h"
+
+#include "mb/pg_wchar.h"
+
+#ifdef HAVE_REPLICATION_ORIGINS
+#include "replication/origin.h"
+#endif
+
+#include "utils/builtins.h"
+#include "utils/json.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/typcache.h"
+
+
+/*
+ * Write BEGIN to the output stream.
+ */
+void
+pglogical_json_write_begin(StringInfo out, PGLogicalOutputData *data, ReorderBufferTXN *txn)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfoString(out, "\"action\":\"B\"");
+ appendStringInfo(out, ", \"has_catalog_changes\":\"%c\"",
+ txn->has_catalog_changes ? 't' : 'f');
+#ifdef HAVE_REPLICATION_ORIGINS
+ if (txn->origin_id != InvalidRepOriginId)
+ appendStringInfo(out, ", \"origin_id\":\"%u\"", txn->origin_id);
+#endif
+ if (!data->client_no_txinfo)
+ {
+ appendStringInfo(out, ", \"xid\":\"%u\"", txn->xid);
+ appendStringInfo(out, ", \"first_lsn\":\"%X/%X\"",
+ (uint32)(txn->first_lsn >> 32), (uint32)(txn->first_lsn));
+#ifdef HAVE_REPLICATION_ORIGINS
+ appendStringInfo(out, ", \"origin_lsn\":\"%X/%X\"",
+ (uint32)(txn->origin_lsn >> 32), (uint32)(txn->origin_lsn));
+#endif
+ if (txn->commit_time != 0)
+ appendStringInfo(out, ", \"commit_time\":\"%s\"",
+ timestamptz_to_str(txn->commit_time));
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write COMMIT to the output stream.
+ */
+void
+pglogical_json_write_commit(StringInfo out, PGLogicalOutputData *data, ReorderBufferTXN *txn,
+ XLogRecPtr commit_lsn)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfoString(out, "\"action\":\"C\"");
+ if (!data->client_no_txinfo)
+ {
+ appendStringInfo(out, ", \"final_lsn\":\"%X/%X\"",
+ (uint32)(txn->final_lsn >> 32), (uint32)(txn->final_lsn));
+ appendStringInfo(out, ", \"end_lsn\":\"%X/%X\"",
+ (uint32)(txn->end_lsn >> 32), (uint32)(txn->end_lsn));
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write a tuple to the outputstream, in the most efficient format possible.
+ */
+static void
+json_write_tuple(StringInfo out, Relation rel, HeapTuple tuple)
+{
+ TupleDesc desc;
+ Datum tupdatum,
+ json;
+
+ desc = RelationGetDescr(rel);
+ tupdatum = heap_copy_tuple_as_datum(tuple, desc);
+ json = DirectFunctionCall1(row_to_json, tupdatum);
+
+ appendStringInfoString(out, TextDatumGetCString(json));
+}
+
+/*
+ * Write change.
+ *
+ * Generic function handling DML changes.
+ */
+static void
+pglogical_json_write_change(StringInfo out, const char *change, Relation rel,
+ HeapTuple oldtuple, HeapTuple newtuple)
+{
+ appendStringInfoChar(out, '{');
+ appendStringInfo(out, "\"action\":\"%s\",\"relation\":[\"%s\",\"%s\"]",
+ change,
+ get_namespace_name(RelationGetNamespace(rel)),
+ RelationGetRelationName(rel));
+
+ if (oldtuple)
+ {
+ appendStringInfoString(out, ",\"oldtuple\":");
+ json_write_tuple(out, rel, oldtuple);
+ }
+ if (newtuple)
+ {
+ appendStringInfoString(out, ",\"newtuple\":");
+ json_write_tuple(out, rel, newtuple);
+ }
+ appendStringInfoChar(out, '}');
+}
+
+/*
+ * Write INSERT to the output stream.
+ */
+void
+pglogical_json_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple)
+{
+ pglogical_json_write_change(out, "I", rel, NULL, newtuple);
+}
+
+/*
+ * Write UPDATE to the output stream.
+ */
+void
+pglogical_json_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple)
+{
+ pglogical_json_write_change(out, "U", rel, oldtuple, newtuple);
+}
+
+/*
+ * Write DELETE to the output stream.
+ */
+void
+pglogical_json_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple)
+{
+ pglogical_json_write_change(out, "D", rel, oldtuple, NULL);
+}
+
+/*
+ * The startup message should be constructed as a json object, one
+ * key/value per DefElem list member.
+ */
+void
+json_write_startup_message(StringInfo out, List *msg)
+{
+ ListCell *lc;
+ bool first = true;
+
+ appendStringInfoString(out, "{\"action\":\"S\", \"params\": {");
+ foreach (lc, msg)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ if (first)
+ first = false;
+ else
+ appendStringInfoChar(out, ',');
+ escape_json(out, param->defname);
+ appendStringInfoChar(out, ':');
+ escape_json(out, strVal(param->arg));
+ }
+ appendStringInfoString(out, "}}");
+}
diff --git a/contrib/pglogical_output/pglogical_proto_json.h b/contrib/pglogical_output/pglogical_proto_json.h
new file mode 100644
index 0000000..d853e9e
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_json.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_json.h
+ * pglogical protocol, json implementation
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_json.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_JSON_H
+#define PG_LOGICAL_PROTO_JSON_H
+
+
+extern void pglogical_json_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+extern void pglogical_json_write_commit(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+extern void pglogical_json_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+extern void pglogical_json_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+extern void pglogical_json_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+extern void json_write_startup_message(StringInfo out, List *msg);
+
+#endif /* PG_LOGICAL_PROTO_JSON_H */
diff --git a/contrib/pglogical_output/pglogical_proto_native.c b/contrib/pglogical_output/pglogical_proto_native.c
new file mode 100644
index 0000000..2dfad8b
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_native.c
@@ -0,0 +1,513 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_native.c
+ * pglogical binary protocol functions
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_native.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "miscadmin.h"
+
+#include "pglogical_output.h"
+#include "pglogical_relmetacache.h"
+#include "pglogical_proto_native.h"
+
+#include "access/sysattr.h"
+#include "access/tuptoaster.h"
+#include "access/xact.h"
+
+#include "catalog/catversion.h"
+#include "catalog/index.h"
+
+#include "catalog/namespace.h"
+#include "catalog/pg_class.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_namespace.h"
+#include "catalog/pg_type.h"
+
+#include "commands/dbcommands.h"
+
+#include "executor/spi.h"
+
+#include "libpq/pqformat.h"
+
+#include "mb/pg_wchar.h"
+
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/typcache.h"
+
+#define IS_REPLICA_IDENTITY 1
+
+static void pglogical_write_attrs(StringInfo out, Relation rel);
+static void pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple);
+static char decide_datum_transfer(Form_pg_attribute att,
+ Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes);
+
+/*
+ * Write relation description to the output stream.
+ */
+void
+pglogical_write_rel(StringInfo out, PGLogicalOutputData *data, Relation rel,
+ struct PGLRelMetaCacheEntry *cache_entry)
+{
+ const char *nspname;
+ uint8 nspnamelen;
+ const char *relname;
+ uint8 relnamelen;
+ uint8 flags = 0;
+
+ /* must not have cache entry if metacache off; must have entry if on */
+ Assert( (data->relmeta_cache_size == 0) == (cache_entry == NULL) );
+ /* if cache enabled must never be called with an already-cached rel */
+ Assert(cache_entry == NULL || !cache_entry->is_cached);
+
+ pq_sendbyte(out, 'R'); /* sending RELATION */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ nspname = get_namespace_name(rel->rd_rel->relnamespace);
+ if (nspname == NULL)
+ elog(ERROR, "cache lookup failed for namespace %u",
+ rel->rd_rel->relnamespace);
+ nspnamelen = strlen(nspname) + 1;
+
+ relname = NameStr(rel->rd_rel->relname);
+ relnamelen = strlen(relname) + 1;
+
+ pq_sendbyte(out, nspnamelen); /* schema name length */
+ pq_sendbytes(out, nspname, nspnamelen);
+
+ pq_sendbyte(out, relnamelen); /* table name length */
+ pq_sendbytes(out, relname, relnamelen);
+
+ /* send the attribute info */
+ pglogical_write_attrs(out, rel);
+
+ /*
+ * Since we've sent the whole relation metadata not just the columns for
+ * the coming row(s), we can omit sending it again. The client will cache
+ * it. If the relation changes the cached flag is cleared by
+ * pglogical_output and we'll be called again next time it's touched.
+ *
+ * We don't care about the cache size here, the size management is done
+ * in the generic cache code.
+ */
+ if (cache_entry != NULL)
+ cache_entry->is_cached = true;
+}
+
+/*
+ * Write relation attributes to the outputstream.
+ */
+static void
+pglogical_write_attrs(StringInfo out, Relation rel)
+{
+ TupleDesc desc;
+ int i;
+ uint16 nliveatts = 0;
+ Bitmapset *idattrs;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'A'); /* sending ATTRS */
+
+ /* send number of live attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* fetch bitmap of REPLICATION IDENTITY attributes */
+ idattrs = RelationGetIndexAttrBitmap(rel, INDEX_ATTR_BITMAP_IDENTITY_KEY);
+
+ /* send the attributes */
+ for (i = 0; i < desc->natts; i++)
+ {
+ Form_pg_attribute att = desc->attrs[i];
+ uint8 flags = 0;
+ uint16 len;
+ const char *attname;
+
+ if (att->attisdropped)
+ continue;
+
+ if (bms_is_member(att->attnum - FirstLowInvalidHeapAttributeNumber,
+ idattrs))
+ flags |= IS_REPLICA_IDENTITY;
+
+ pq_sendbyte(out, 'C'); /* column definition follows */
+ pq_sendbyte(out, flags);
+
+ pq_sendbyte(out, 'N'); /* column name block follows */
+ attname = NameStr(att->attname);
+ len = strlen(attname) + 1;
+ pq_sendint(out, len, 2);
+ pq_sendbytes(out, attname, len); /* data */
+ }
+}
+
+/*
+ * Write BEGIN to the output stream.
+ */
+void
+pglogical_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'B'); /* BEGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, txn->final_lsn);
+ pq_sendint64(out, txn->commit_time);
+ pq_sendint(out, txn->xid, 4);
+}
+
+/*
+ * Write COMMIT to the output stream.
+ */
+void
+pglogical_write_commit(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'C'); /* sending COMMIT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* send fixed fields */
+ pq_sendint64(out, commit_lsn);
+ pq_sendint64(out, txn->end_lsn);
+ pq_sendint64(out, txn->commit_time);
+}
+
+/*
+ * Write ORIGIN to the output stream.
+ */
+void
+pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn)
+{
+ uint8 flags = 0;
+ uint8 len;
+
+ Assert(strlen(origin) < 255);
+
+ pq_sendbyte(out, 'O'); /* ORIGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
+
+ /* fixed fields */
+ pq_sendint64(out, origin_lsn);
+
+ /* origin */
+ len = strlen(origin) + 1;
+ pq_sendbyte(out, len);
+ pq_sendbytes(out, origin, len);
+}
+
+/*
+ * Write INSERT to the output stream.
+ */
+void
+pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'I'); /* action INSERT */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write UPDATE to the output stream.
+ */
+void
+pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple, HeapTuple newtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'U'); /* action UPDATE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ if (oldtuple != NULL)
+ {
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+ }
+
+ pq_sendbyte(out, 'N'); /* new tuple follows */
+ pglogical_write_tuple(out, data, rel, newtuple);
+}
+
+/*
+ * Write DELETE to the output stream.
+ */
+void
+pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple)
+{
+ uint8 flags = 0;
+
+ pq_sendbyte(out, 'D'); /* action DELETE */
+
+ /* send the flags field */
+ pq_sendbyte(out, flags);
+
+ /* use Oid as relation identifier */
+ pq_sendint(out, RelationGetRelid(rel), 4);
+
+ /* FIXME support whole tuple (O tuple type) */
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+}
+
+/*
+ * Most of the brains for startup message creation lives in
+ * pglogical_config.c, so this presently just sends the set of key/value pairs.
+ */
+void
+write_startup_message(StringInfo out, List *msg)
+{
+ ListCell *lc;
+
+ pq_sendbyte(out, 'S'); /* message type field */
+ pq_sendbyte(out, 1); /* startup message version */
+ foreach (lc, msg)
+ {
+ DefElem *param = (DefElem*)lfirst(lc);
+ Assert(IsA(param->arg, String) && strVal(param->arg) != NULL);
+ /* null-terminated key and value pairs, in client_encoding */
+ pq_sendstring(out, param->defname);
+ pq_sendstring(out, strVal(param->arg));
+ }
+}
+
+/*
+ * Write a tuple to the outputstream, in the most efficient format possible.
+ */
+static void
+pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple tuple)
+{
+ TupleDesc desc;
+ Datum values[MaxTupleAttributeNumber];
+ bool isnull[MaxTupleAttributeNumber];
+ int i;
+ uint16 nliveatts = 0;
+
+ desc = RelationGetDescr(rel);
+
+ pq_sendbyte(out, 'T'); /* sending TUPLE */
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
+
+ /* try to allocate enough memory from the get go */
+ enlargeStringInfo(out, tuple->t_len +
+ nliveatts * (1 + 4));
+
+ /*
+ * XXX: should this prove to be a relevant bottleneck, it might be
+ * interesting to inline heap_deform_tuple() here, we don't actually need
+ * the information in the form we get from it.
+ */
+ heap_deform_tuple(tuple, desc, values, isnull);
+
+ for (i = 0; i < desc->natts; i++)
+ {
+ HeapTuple typtup;
+ Form_pg_type typclass;
+ Form_pg_attribute att = desc->attrs[i];
+ char transfer_type;
+
+ /* skip dropped columns */
+ if (att->attisdropped)
+ continue;
+
+ if (isnull[i])
+ {
+ pq_sendbyte(out, 'n'); /* null column */
+ continue;
+ }
+ else if (att->attlen == -1 && VARATT_IS_EXTERNAL_ONDISK(values[i]))
+ {
+ pq_sendbyte(out, 'u'); /* unchanged toast column */
+ continue;
+ }
+
+ typtup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid));
+ if (!HeapTupleIsValid(typtup))
+ elog(ERROR, "cache lookup failed for type %u", att->atttypid);
+ typclass = (Form_pg_type) GETSTRUCT(typtup);
+
+ transfer_type = decide_datum_transfer(att, typclass,
+ data->allow_internal_basetypes,
+ data->allow_binary_basetypes);
+
+ switch (transfer_type)
+ {
+ case 'i':
+ pq_sendbyte(out, 'i'); /* internal-format binary data follows */
+
+ /* pass by value */
+ if (att->attbyval)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ enlargeStringInfo(out, att->attlen);
+ store_att_byval(out->data + out->len, values[i],
+ att->attlen);
+ out->len += att->attlen;
+ out->data[out->len] = '\0';
+ }
+ /* fixed length non-varlena pass-by-reference type */
+ else if (att->attlen > 0)
+ {
+ pq_sendint(out, att->attlen, 4); /* length */
+
+ appendBinaryStringInfo(out, DatumGetPointer(values[i]),
+ att->attlen);
+ }
+ /* varlena type */
+ else if (att->attlen == -1)
+ {
+ char *data = DatumGetPointer(values[i]);
+
+ /* send indirect datums inline */
+ if (VARATT_IS_EXTERNAL_INDIRECT(values[i]))
+ {
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, data);
+ data = (char *) redirect.pointer;
+ }
+
+ Assert(!VARATT_IS_EXTERNAL(data));
+
+ pq_sendint(out, VARSIZE_ANY(data), 4); /* length */
+
+ appendBinaryStringInfo(out, data, VARSIZE_ANY(data));
+ }
+ else
+ elog(ERROR, "unsupported tuple type");
+
+ break;
+
+ case 'b':
+ {
+ bytea *outputbytes;
+ int len;
+
+ pq_sendbyte(out, 'b'); /* binary send/recv data follows */
+
+ outputbytes = OidSendFunctionCall(typclass->typsend,
+ values[i]);
+
+ len = VARSIZE(outputbytes) - VARHDRSZ;
+ pq_sendint(out, len, 4); /* length */
+ pq_sendbytes(out, VARDATA(outputbytes), len); /* data */
+ pfree(outputbytes);
+ }
+ break;
+
+ default:
+ {
+ char *outputstr;
+ int len;
+
+ pq_sendbyte(out, 't'); /* 'text' data follows */
+
+ outputstr = OidOutputFunctionCall(typclass->typoutput,
+ values[i]);
+ len = strlen(outputstr) + 1;
+ pq_sendint(out, len, 4); /* length */
+ appendBinaryStringInfo(out, outputstr, len); /* data */
+ pfree(outputstr);
+ }
+ }
+
+ ReleaseSysCache(typtup);
+ }
+}
+
+/*
+ * Make the executive decision about which protocol to use.
+ */
+static char
+decide_datum_transfer(Form_pg_attribute att, Form_pg_type typclass,
+ bool allow_internal_basetypes,
+ bool allow_binary_basetypes)
+{
+ /*
+ * Use the binary protocol, if allowed, for builtin & plain datatypes.
+ */
+ if (allow_internal_basetypes &&
+ typclass->typtype == 'b' &&
+ att->atttypid < FirstNormalObjectId &&
+ typclass->typelem == InvalidOid)
+ {
+ return 'i';
+ }
+ /*
+ * Use send/recv, if allowed, if the type is plain or builtin.
+ *
+ * XXX: we can't use send/recv for array or composite types for now due to
+ * the embedded oids.
+ */
+ else if (allow_binary_basetypes &&
+ OidIsValid(typclass->typreceive) &&
+ (att->atttypid < FirstNormalObjectId || typclass->typtype != 'c') &&
+ (att->atttypid < FirstNormalObjectId || typclass->typelem == InvalidOid))
+ {
+ return 'b';
+ }
+
+ return 't';
+}
diff --git a/contrib/pglogical_output/pglogical_proto_native.h b/contrib/pglogical_output/pglogical_proto_native.h
new file mode 100644
index 0000000..b8433e9
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_proto_native.h
@@ -0,0 +1,38 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_proto_native.h
+ * pglogical protocol, native implementation
+ *
+ * Copyright (c) 2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_proto_native.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_LOGICAL_PROTO_NATIVE_H
+#define PG_LOGICAL_PROTO_NATIVE_H
+
+
+extern void pglogical_write_rel(StringInfo out, PGLogicalOutputData *data, Relation rel,
+ struct PGLRelMetaCacheEntry *cache_entry);
+
+extern void pglogical_write_begin(StringInfo out, PGLogicalOutputData *data,
+ ReorderBufferTXN *txn);
+extern void pglogical_write_commit(StringInfo out,PGLogicalOutputData *data,
+ ReorderBufferTXN *txn, XLogRecPtr commit_lsn);
+
+extern void pglogical_write_origin(StringInfo out, const char *origin,
+ XLogRecPtr origin_lsn);
+
+extern void pglogical_write_insert(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple newtuple);
+extern void pglogical_write_update(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple,
+ HeapTuple newtuple);
+extern void pglogical_write_delete(StringInfo out, PGLogicalOutputData *data,
+ Relation rel, HeapTuple oldtuple);
+
+extern void write_startup_message(StringInfo out, List *msg);
+
+#endif /* PG_LOGICAL_PROTO_NATIVE_H */
diff --git a/contrib/pglogical_output/pglogical_relmetacache.c b/contrib/pglogical_output/pglogical_relmetacache.c
new file mode 100644
index 0000000..5b4426f
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_relmetacache.c
@@ -0,0 +1,194 @@
+/*-------------------------------------------------------------------------
+ *
+ * pglogical_relmetacache.c
+ * Logical Replication relmetacache plugin
+ *
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * pglogical_relmetacache.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "pglogical_output.h"
+#include "pglogical_relmetacache.h"
+
+#include "utils/catcache.h"
+#include "utils/inval.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+
+static void relmeta_cache_callback(Datum arg, Oid relid);
+
+/*
+ * We need a global hash table that invalidation callbacks can
+ * access because they survive past the logical decoding context and
+ * therefore past our local PGLogicalOutputData's lifetime when
+ * using the SQL interface. We cannot just pass them a pointer to a
+ * palloc'd struct.
+ */
+static HTAB *RelMetaCache = NULL;
+
+
+/*
+ * Initialize the relation metadata cache if not already initialized.
+ *
+ * Purge it if it already exists.
+ *
+ * The hash table its self must be in CacheMemoryContext or TopMemoryContext
+ * since it persists outside the decoding session.
+ */
+void
+pglogical_init_relmetacache(void)
+{
+ HASHCTL ctl;
+
+ if (RelMetaCache == NULL)
+ {
+ /* first time, init the cache */
+ int hash_flags = HASH_ELEM | HASH_CONTEXT;
+
+ /* Make sure we've initialized CacheMemoryContext. */
+ if (CacheMemoryContext == NULL)
+ CreateCacheMemoryContext();
+
+ MemSet(&ctl, 0, sizeof(ctl));
+ ctl.keysize = sizeof(Oid);
+ ctl.entrysize = sizeof(struct PGLRelMetaCacheEntry);
+ /* safe to allocate to CacheMemoryContext since it's never reset */
+ ctl.hcxt = CacheMemoryContext;
+
+#if PG_VERSION_NUM >= 90500
+ hash_flags |= HASH_BLOBS;
+#else
+ ctl.hash = tag_hash;
+ hash_flags |= HASH_FUNCTION;
+#endif
+
+ RelMetaCache = hash_create("pglogical relation metadata cache", 128,
+ &ctl, hash_flags);
+
+ Assert(RelMetaCache != NULL);
+
+ /*
+ * Watch for invalidation events.
+ *
+ * We don't pass PGLogicalOutputData here because it's scoped to the
+ * individual decoding session, which with the SQL interface has a shorter
+ * lifetime than the relcache invalidation callback registration. We have
+ * no way to remove invalidation callbacks at the end of the decoding
+ * session so we have to cope with them being called later.
+ */
+ CacheRegisterRelcacheCallback(relmeta_cache_callback, (Datum)0);
+ }
+ else
+ {
+ /*
+ * On re-init we must flush the cache since there could be
+ * dangling pointers to api_private data in the freed
+ * decoding context of a prior session. We could go through
+ * and clear them and the is_cached flag but it seems best
+ * to have a clean slate.
+ */
+ HASH_SEQ_STATUS status;
+ struct PGLRelMetaCacheEntry *hentry;
+ hash_seq_init(&status, RelMetaCache);
+
+ while ((hentry = (struct PGLRelMetaCacheEntry*) hash_seq_search(&status)) != NULL)
+ {
+ if (hash_search(RelMetaCache,
+ (void *) &hentry->relid,
+ HASH_REMOVE, NULL) == NULL)
+ elog(ERROR, "pglogical RelMetaCache hash table corrupted");
+ }
+
+ return;
+ }
+}
+
+/*
+ * Relation metadata invalidation, for when a relcache invalidation
+ * means that we need to resend table metadata to the client.
+ */
+static void
+relmeta_cache_callback(Datum arg, Oid relid)
+ {
+ /*
+ * Nobody keeps pointers to entries in this hash table around so
+ * it's safe to directly HASH_REMOVE the entries as soon as they are
+ * invalidated. Finding them and flagging them invalid then removing
+ * them lazily might save some memory churn for tables that get
+ * repeatedly invalidated and re-sent, but it dodesn't seem worth
+ * doing.
+ *
+ * Getting invalidations for relations that aren't in the table is
+ * entirely normal, since there's no way to unregister for an
+ * invalidation event. So we don't care if it's found or not.
+ */
+ (void) hash_search(RelMetaCache, &relid, HASH_REMOVE, NULL);
+ }
+
+/*
+ * Look up an entry, creating it not found.
+ *
+ * Newly created entries are returned as is_cached=false. The API
+ * hook can set is_cached to skip subsequent updates if it sent a
+ * complete response that the client will cache.
+ *
+ * Returns true on a cache hit, false on a miss.
+ */
+bool
+pglogical_cache_relmeta(struct PGLogicalOutputData *data,
+ Relation rel, struct PGLRelMetaCacheEntry **entry)
+{
+ struct PGLRelMetaCacheEntry *hentry;
+ bool found;
+
+ if (data->relmeta_cache_size == 0)
+ {
+ /*
+ * If cache is disabled must treat every search as a miss
+ * and return no entry to populate.
+ */
+ *entry = NULL;
+ return false;
+ }
+
+ /* Find cached function info, creating if not found */
+ hentry = (struct PGLRelMetaCacheEntry*) hash_search(RelMetaCache,
+ (void *)(&RelationGetRelid(rel)),
+ HASH_ENTER, &found);
+
+ if (!found)
+ {
+ Assert(hentry->relid = RelationGetRelid(rel));
+ hentry->is_cached = false;
+ hentry->api_private = NULL;
+ }
+
+ Assert(hentry != NULL);
+
+ *entry = hentry;
+ return hentry->is_cached;
+}
+
+
+/*
+ * Tear down the relation metadata cache.
+ *
+ * Do *not* call this at decoding shutdown. The hash table must
+ * continue to exist so that relcache invalidation callbacks can
+ * continue to reference it after a SQL decoding session finishes.
+ * It must be called at backend shutdown only.
+ */
+void
+pglogical_destroy_relmetacache(void)
+{
+ if (RelMetaCache != NULL)
+ {
+ hash_destroy(RelMetaCache);
+ RelMetaCache = NULL;
+ }
+}
diff --git a/contrib/pglogical_output/pglogical_relmetacache.h b/contrib/pglogical_output/pglogical_relmetacache.h
new file mode 100644
index 0000000..b5e8c01
--- /dev/null
+++ b/contrib/pglogical_output/pglogical_relmetacache.h
@@ -0,0 +1,19 @@
+#ifndef PGLOGICAL_RELMETA_CACHE_H
+#define PGLOGICAL_RELMETA_CACHE_H
+
+struct PGLRelMetaCacheEntry
+{
+ Oid relid;
+ /* Does the client have this relation cached? */
+ bool is_cached;
+ /* Field for API plugin use, must be alloc'd in decoding context */
+ void *api_private;
+};
+
+struct PGLogicalOutputData;
+
+extern void pglogical_init_relmetacache(void);
+extern bool pglogical_cache_relmeta(struct PGLogicalOutputData *data, Relation rel, struct PGLRelMetaCacheEntry **entry);
+extern void pglogical_destroy_relmetacache(void);
+
+#endif /* PGLOGICAL_RELMETA_CACHE_H */
diff --git a/contrib/pglogical_output/regression.conf b/contrib/pglogical_output/regression.conf
new file mode 100644
index 0000000..367f706
--- /dev/null
+++ b/contrib/pglogical_output/regression.conf
@@ -0,0 +1,2 @@
+wal_level = logical
+max_replication_slots = 4
diff --git a/contrib/pglogical_output/sql/basic_json.sql b/contrib/pglogical_output/sql/basic_json.sql
new file mode 100644
index 0000000..e8a2352
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_json.sql
@@ -0,0 +1,24 @@
+\i sql/basic_setup.sql
+
+-- Simple decode with text-format tuples
+TRUNCATE TABLE json_decoding_output;
+
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+SELECT * FROM get_startup_params();
+SELECT * FROM get_queued_data();
+
+TRUNCATE TABLE json_decoding_output;
+
+\i sql/basic_teardown.sql
diff --git a/contrib/pglogical_output/sql/basic_native.sql b/contrib/pglogical_output/sql/basic_native.sql
new file mode 100644
index 0000000..6f1862b
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_native.sql
@@ -0,0 +1,37 @@
+\i sql/basic_setup.sql
+
+-- Simple decode with text-format tuples
+--
+-- It's still the logical decoding binary protocol and as such it has
+-- embedded timestamps, and pglogical its self has embedded LSNs, xids,
+-- etc. So all we can really do is say "yup, we got the expected number
+-- of messages".
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- ... and send/recv binary format
+-- The main difference visible is that the bytea fields aren't encoded
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'binary.want_binary_basetypes', '1',
+ 'binary.basetypes_major_version', (current_setting('server_version_num')::integer / 100)::text);
+
+-- Now enable the relation metadata cache and verify that we get the expected
+-- reduction in number of messages. Not much else we can look for.
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'relmeta_cache_size', '-1');
+
+\i sql/basic_teardown.sql
diff --git a/contrib/pglogical_output/sql/basic_setup.sql b/contrib/pglogical_output/sql/basic_setup.sql
new file mode 100644
index 0000000..19e154c
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_setup.sql
@@ -0,0 +1,62 @@
+SET synchronous_commit = on;
+
+-- Schema setup
+
+CREATE TABLE demo (
+ seq serial primary key,
+ tx text,
+ ts timestamp,
+ jsb jsonb,
+ js json,
+ ba bytea
+);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Queue up some work to decode with a variety of types
+
+INSERT INTO demo(tx) VALUES ('textval');
+INSERT INTO demo(ba) VALUES (BYTEA '\xDEADBEEF0001');
+INSERT INTO demo(ts, tx) VALUES (TIMESTAMP '2045-09-12 12:34:56.00', 'blah');
+INSERT INTO demo(js, jsb) VALUES ('{"key":"value"}', '{"key":"value"}');
+
+-- Rolled back txn
+BEGIN;
+DELETE FROM demo;
+INSERT INTO demo(tx) VALUES ('blahblah');
+ROLLBACK;
+
+-- Multi-statement transaction with subxacts
+BEGIN;
+SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('row1');
+RELEASE SAVEPOINT sp1;
+SAVEPOINT sp2;
+UPDATE demo SET tx = 'update-rollback' WHERE tx = 'row1';
+ROLLBACK TO SAVEPOINT sp2;
+SAVEPOINT sp3;
+INSERT INTO demo(tx) VALUES ('row2');
+INSERT INTO demo(tx) VALUES ('row3');
+RELEASE SAVEPOINT sp3;
+SAVEPOINT sp4;
+DELETE FROM demo WHERE tx = 'row2';
+RELEASE SAVEPOINT sp4;
+SAVEPOINT sp5;
+UPDATE demo SET tx = 'updated' WHERE tx = 'row1';
+COMMIT;
+
+
+-- txn with catalog changes
+BEGIN;
+CREATE TABLE cat_test(id integer);
+INSERT INTO cat_test(id) VALUES (42);
+COMMIT;
+
+-- Aborted subxact with catalog changes
+BEGIN;
+INSERT INTO demo(tx) VALUES ('1');
+SAVEPOINT sp1;
+ALTER TABLE demo DROP COLUMN tx;
+ROLLBACK TO SAVEPOINT sp1;
+INSERT INTO demo(tx) VALUES ('2');
+COMMIT;
diff --git a/contrib/pglogical_output/sql/basic_teardown.sql b/contrib/pglogical_output/sql/basic_teardown.sql
new file mode 100644
index 0000000..d7a752f
--- /dev/null
+++ b/contrib/pglogical_output/sql/basic_teardown.sql
@@ -0,0 +1,4 @@
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE demo;
+DROP TABLE cat_test;
diff --git a/contrib/pglogical_output/sql/cleanup.sql b/contrib/pglogical_output/sql/cleanup.sql
new file mode 100644
index 0000000..e7a02c8
--- /dev/null
+++ b/contrib/pglogical_output/sql/cleanup.sql
@@ -0,0 +1,4 @@
+DROP TABLE excluded_startup_keys;
+DROP TABLE json_decoding_output;
+DROP FUNCTION get_queued_data();
+DROP FUNCTION get_startup_params();
diff --git a/contrib/pglogical_output/sql/encoding_json.sql b/contrib/pglogical_output/sql/encoding_json.sql
new file mode 100644
index 0000000..543c306
--- /dev/null
+++ b/contrib/pglogical_output/sql/encoding_json.sql
@@ -0,0 +1,58 @@
+SET synchronous_commit = on;
+
+-- This file doesn't share common setup with the native tests,
+-- since it's specific to how the text protocol handles encodings.
+
+CREATE TABLE enctest(blah text);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+
+SET client_encoding = 'UTF-8';
+INSERT INTO enctest(blah)
+VALUES
+('áàä'),('fl'), ('½⅓'), ('カンジ');
+RESET client_encoding;
+
+
+SET client_encoding = 'LATIN-1';
+
+-- Will ERROR, explicit encoding request doesn't match client_encoding
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+-- Will succeed since we don't request any encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+-- Will succeed since it matches the current encoding
+-- then ERROR because it can't turn the kanjii into latin-1
+SELECT data
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'LATIN-1',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+RESET client_encoding;
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE enctest;
diff --git a/contrib/pglogical_output/sql/extension.sql b/contrib/pglogical_output/sql/extension.sql
new file mode 100644
index 0000000..00937bf
--- /dev/null
+++ b/contrib/pglogical_output/sql/extension.sql
@@ -0,0 +1,7 @@
+CREATE EXTENSION pglogical_output;
+
+SELECT pglogical_output_proto_version();
+
+SELECT pglogical_output_min_proto_version();
+
+DROP EXTENSION pglogical_output;
diff --git a/contrib/pglogical_output/sql/hooks_json.sql b/contrib/pglogical_output/sql/hooks_json.sql
new file mode 100644
index 0000000..cd58960
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_json.sql
@@ -0,0 +1,49 @@
+\i sql/hooks_setup.sql
+
+
+-- Test table filter
+TRUNCATE TABLE json_decoding_output;
+
+INSERT INTO json_decoding_output(ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+SELECT * FROM get_startup_params();
+SELECT * FROM get_queued_data();
+
+-- test action filter
+TRUNCATE TABLE json_decoding_output;
+
+INSERT INTO json_decoding_output (ch, rn)
+SELECT
+ data::jsonb,
+ row_number() OVER ()
+FROM pg_logical_slot_peek_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter',
+ 'proto_format', 'json',
+ 'no_txinfo', 't');
+
+SELECT * FROM get_startup_params();
+SELECT * FROM get_queued_data();
+
+TRUNCATE TABLE json_decoding_output;
+
+\i sql/hooks_teardown.sql
diff --git a/contrib/pglogical_output/sql/hooks_native.sql b/contrib/pglogical_output/sql/hooks_native.sql
new file mode 100644
index 0000000..e2bfc54
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_native.sql
@@ -0,0 +1,48 @@
+\i sql/hooks_setup.sql
+
+-- Regular hook setup
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_filter',
+ 'pglo_plhooks.client_hook_arg', 'foo'
+ );
+
+-- Test action filter
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.test_action_filter'
+ );
+
+-- Invalid row fiter hook function
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.nosuchfunction'
+ );
+
+-- Hook filter functoin with wrong signature
+SELECT count(data) FROM pg_logical_slot_peek_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'hooks.setup_function', 'public.pglo_plhooks_setup_fn',
+ 'pglo_plhooks.row_filter_hook', 'public.wrong_signature_fn'
+ );
+
+\i sql/hooks_teardown.sql
diff --git a/contrib/pglogical_output/sql/hooks_setup.sql b/contrib/pglogical_output/sql/hooks_setup.sql
new file mode 100644
index 0000000..4de15b7
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_setup.sql
@@ -0,0 +1,37 @@
+CREATE EXTENSION pglogical_output_plhooks;
+
+CREATE FUNCTION test_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ IF nodeid <> 'foo' THEN
+ RAISE EXCEPTION 'Expected nodeid <foo>, got <%>',nodeid;
+ END IF;
+ RETURN relid::regclass::text NOT LIKE '%_filter%';
+END
+$$;
+
+CREATE FUNCTION test_action_filter(relid regclass, action "char", nodeid text)
+returns bool stable language plpgsql AS $$
+BEGIN
+ RETURN action NOT IN ('U', 'D');
+END
+$$;
+
+CREATE FUNCTION wrong_signature_fn(relid regclass)
+returns bool stable language plpgsql as $$
+BEGIN
+END;
+$$;
+
+CREATE TABLE test_filter(id integer);
+CREATE TABLE test_nofilt(id integer);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+INSERT INTO test_filter(id) SELECT generate_series(1,10);
+INSERT INTO test_nofilt(id) SELECT generate_series(1,10);
+
+DELETE FROM test_filter WHERE id % 2 = 0;
+DELETE FROM test_nofilt WHERE id % 2 = 0;
+UPDATE test_filter SET id = id*100 WHERE id = 5;
+UPDATE test_nofilt SET id = id*100 WHERE id = 5;
diff --git a/contrib/pglogical_output/sql/hooks_teardown.sql b/contrib/pglogical_output/sql/hooks_teardown.sql
new file mode 100644
index 0000000..837e2d0
--- /dev/null
+++ b/contrib/pglogical_output/sql/hooks_teardown.sql
@@ -0,0 +1,10 @@
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
+
+DROP TABLE test_filter;
+DROP TABLE test_nofilt;
+
+DROP FUNCTION test_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION test_action_filter(relid regclass, action "char", nodeid text);
+DROP FUNCTION wrong_signature_fn(relid regclass);
+
+DROP EXTENSION pglogical_output_plhooks;
diff --git a/contrib/pglogical_output/sql/params_native.sql b/contrib/pglogical_output/sql/params_native.sql
new file mode 100644
index 0000000..9203bd7
--- /dev/null
+++ b/contrib/pglogical_output/sql/params_native.sql
@@ -0,0 +1,104 @@
+SET synchronous_commit = on;
+
+-- no need to CREATE EXTENSION as we intentionally don't have any catalog presence
+-- Instead, just create a slot.
+
+SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'pglogical_output');
+
+-- Minimal invocation with no data
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+--
+-- Various invalid parameter combos:
+--
+
+-- Text mode is not supported for native protocol
+SELECT data FROM pg_logical_slot_get_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- error, only supports proto v1
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '2',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- error, unrecognised startup params format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '2');
+
+-- Should be OK and result in proto version 1 selection, though we won't
+-- see that here.
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '2',
+ 'startup_params_format', '1');
+
+-- no such encoding / encoding mismatch
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'bork',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- Different spellings of encodings are OK too
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF-8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1');
+
+-- bogus param format
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'invalid');
+
+-- native params format explicitly
+SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'proto_format', 'native');
+
+-- relmeta cache with fixed size (not supported yet, so error)
+SELECT count(data) FROM pg_logical_slot_get_binary_changes('regression_slot',
+ NULL, NULL,
+ 'expected_encoding', 'UTF8',
+ 'min_proto_version', '1',
+ 'max_proto_version', '1',
+ 'startup_params_format', '1',
+ 'relmeta_cache_size', '200');
+
+SELECT 'drop' FROM pg_drop_replication_slot('regression_slot');
diff --git a/contrib/pglogical_output/sql/prep.sql b/contrib/pglogical_output/sql/prep.sql
new file mode 100644
index 0000000..26e79c8
--- /dev/null
+++ b/contrib/pglogical_output/sql/prep.sql
@@ -0,0 +1,30 @@
+CREATE TABLE excluded_startup_keys (key_name text primary key);
+
+INSERT INTO excluded_startup_keys
+VALUES
+('pg_version_num'),('pg_version'),('pg_catversion'),('binary.basetypes_major_version'),('binary.integer_datetimes'),('binary.bigendian'),('binary.maxalign'),('binary.binary_pg_version'),('sizeof_int'),('sizeof_long'),('sizeof_datum');
+
+CREATE UNLOGGED TABLE json_decoding_output(ch jsonb, rn integer);
+
+CREATE OR REPLACE FUNCTION get_startup_params()
+RETURNS TABLE ("key" text, "value" jsonb)
+LANGUAGE sql
+AS $$
+SELECT key, value
+FROM json_decoding_output
+CROSS JOIN LATERAL jsonb_each(ch -> 'params')
+WHERE rn = 1
+ AND key NOT IN (SELECT * FROM excluded_startup_keys)
+ AND ch ->> 'action' = 'S'
+ORDER BY key;
+$$;
+
+CREATE OR REPLACE FUNCTION get_queued_data()
+RETURNS TABLE (data jsonb)
+LANGUAGE sql
+AS $$
+SELECT ch
+FROM json_decoding_output
+WHERE rn > 1
+ORDER BY rn ASC;
+$$;
diff --git a/contrib/pglogical_output_plhooks/.gitignore b/contrib/pglogical_output_plhooks/.gitignore
new file mode 100644
index 0000000..140f8cf
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/.gitignore
@@ -0,0 +1 @@
+*.so
diff --git a/contrib/pglogical_output_plhooks/Makefile b/contrib/pglogical_output_plhooks/Makefile
new file mode 100644
index 0000000..ecd3f89
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/Makefile
@@ -0,0 +1,13 @@
+MODULES = pglogical_output_plhooks
+EXTENSION = pglogical_output_plhooks
+DATA = pglogical_output_plhooks--1.0.sql
+DOCS = README.pglogical_output_plhooks
+
+subdir = contrib/pglogical_output_plhooks
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+
+# Allow the hook plugin to see the pglogical_output headers
+# Necessary because !PGXS builds don't respect PG_CPPFLAGS
+override CPPFLAGS := $(CPPFLAGS) -I$(top_srcdir)/contrib/pglogical_output
diff --git a/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks b/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
new file mode 100644
index 0000000..f2ad9d4
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/README.pglogical_output_plhooks
@@ -0,0 +1,158 @@
+pglogical_output_plhooks is an example module for pglogical_output, showing how
+hooks can be implemented.
+
+It provides C wrappers to allow hooks to be written in any supported PL,
+such as PL/PgSQL.
+
+No effort is made to be efficient. To avoid the need to set up cache
+invalidation handling function calls are done via oid each time, with no
+FmgrInfo caching. Also, memory contexts are reset rather freely. If you
+want efficiency, write your hook in C.
+
+(Catalog timetravel is another reason not to write hooks in PLs; see below).
+
+Simple pointless example
+===
+
+To compile and install, just "make USE_PGXS=1 install". Note that pglogical
+must already be installed so that its headers can be found. You might have
+to set the `PATH` so that `pg_config` can be found.
+
+To use it:
+
+ CREATE EXTENSION pglogical_output_plhooks IN SCHEMA public;
+
+in the target database.
+
+Then create at least one hook procedure, of the supported hooks listed below.
+For the sake of this example we'll use some of the toy examples provided in the
+extension:
+
+* startup function: pglo_plhooks_demo_startup
+* row filter: pglo_plhooks_demo_row_filter
+* txn filter: pglo_plhooks_demo_txn_filter
+* shutdown function: pglo_plhooks_demo_shutdown
+
+Now add some arguments to your pglogical_output client's logical decoding setup
+parameters to specify the hook setup function and to tell
+pglogical_output_plhooks about one or more of the hooks you wish it to run. For
+example you might add the following parameters:
+
+ hooks.setup_function, public.pglo_plhooks_setup_fn,
+ pglo_plhooks.startup_hook, pglo_plhooks_demo_startup,
+ pglo_plhooks.row_filter_hook, pglo_plhooks_demo_row_filter,
+ pglo_plhooks.txn_filter_hook, pglo_plhooks_demo_txn_filter,
+ pglo_plhooks.shutdown_hook, pglo_plhooks_demo_shutdown,
+ pglo_plhooks.client_hook_arg, 'whatever-you-want'
+
+to configure the extension to load its hooks, then configure all the demo hooks.
+
+Why the preference for C hooks?
+===
+
+Speed. The row filter hook is called for *every single row* replicated.
+
+If a hook raises an ERROR then replication will probably stop. You won't be
+able to fix it either, because when you change the hook definition the new
+definition won't be visible in the catalogs at the current replay position due
+to catalog time travel. The old definition that raises an error will keep being
+used. You'll need to remove the problem hook from your logical decoding startup
+parameters, which will disable use the hook entirely, until replay proceeds
+past the point you fixed the problem with the hook function.
+
+Similarly, if you try to add use of a newly defined hook on an existing
+replication slot that hasn't replayed past the point you defined the hook yet,
+you'll get an error complaining that the hook function doesn't exist. Even
+though it clearly does when you look at it in psql. The reason is the same: in
+the time traveled catalogs it really doesn't exist. You have to replay past the
+point the hook was created then enable it. In this case the
+pglogical_output_plhooks startup hook will actually see your functions, but
+fail when it tries to call them during decoding since they'll appear to have
+vanished.
+
+If you write your hooks in C you can redefine them rather more easily, since
+the function definition is not subject to catalog timetravel. More importantly,
+it'll probably be a lot faster. The plhooks code has to do a lot of translation
+to pass information to the PL functions and more to get results back; it also
+has to do a lot of memory allocations and a memory context reset after each
+call. That all adds up.
+
+(You could actually write C functions to be called by this extension, but
+that'd be crazy.)
+
+Available hooks
+===
+
+The four hooks provided by pglogical_output are exposed by the module. See the
+pglogical_output documentation for details on what each hook does and when it
+runs.
+
+A function for each hook must have *exactly* the specified parameters and
+return value, or you'll get an error.
+
+None of the functions may return NULL. If they do you'll get an error.
+
+If you specified `pglo_plhooks.client_hook_arg` in the startup parameters it is
+passed as `client_hook_arg` to all hooks. If not specified the empty string is
+passed.
+
+You can find some toy examples in `pglogical_output_plhooks--1.0.sql`.
+
+
+
+Startup hook
+---
+
+Configured with `pglo_plhooks.startup_hook` startup parameter. Runs when
+logical decoding starts.
+
+Signature *must* be:
+
+ CREATE FUNCTION whatever_funcname(startup_params text[], client_hook_arg text)
+ RETURNS text[]
+
+startup_params is an array of the startup params passed to the pglogical output
+plugin, as alternating key/value elements in text representation.
+
+client_hook_arg is also passed.
+
+The return value is an array of alternating key/value elements forming a set
+of parameters you wish to add to the startup reply message sent by pglogical
+on decoding start. It must not be null; return `ARRAY[]::text[]` if you don't
+want to add any params.
+
+Transaction filter
+---
+
+The arguments are the replication origin identifier and the client hook param.
+
+The return value is true to keep the transaction, false to discard it.
+
+Signature:
+
+ CREATE FUNCTION whatevername(origin_id int, client_hook_arg text)
+ RETURNS boolean
+
+Row filter
+--
+
+Called for each row. Return true to replicate the row, false to discard it.
+
+Arguments are the oid of the affected relation, and the change type: 'I'nsert,
+'U'pdate or 'D'elete. There is no way to access the change data - columns changed,
+new values, etc.
+
+Signature:
+
+ CREATE FUNCTION whatevername(affected_rel regclass, change_type "char", client_hook_arg text)
+ RETURNS boolean
+
+Shutdown hook
+--
+
+Pretty uninteresting, but included for completeness.
+
+Signature:
+
+ CREATE FUNCTION whatevername(client_hook_arg text)
+ RETURNS void
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql b/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
new file mode 100644
index 0000000..cdd2af3
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks--1.0.sql
@@ -0,0 +1,89 @@
+\echo Use "CREATE EXTENSION pglogical_output_plhooks" to load this file. \quit
+
+-- Use @extschema@ or leave search_path unchanged, don't use explicit schema
+
+CREATE FUNCTION pglo_plhooks_setup_fn(internal)
+RETURNS void
+STABLE
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+COMMENT ON FUNCTION pglo_plhooks_setup_fn(internal)
+IS 'Register pglogical output pl hooks. See docs for how to specify functions';
+
+--
+-- Called as the startup hook.
+--
+-- There's no useful way to expose the private data segment, so you
+-- just don't get to use that from pl hooks at this point. The C
+-- wrapper will extract a startup param named pglo_plhooks.client_hook_arg
+-- for you and pass it as client_hook_arg to all callbacks, though.
+--
+-- For implementation convenience, a null client_hook_arg is passed
+-- as the empty string.
+--
+-- Must return the empty array, not NULL, if it has nothing to add.
+--
+CREATE FUNCTION pglo_plhooks_demo_startup(startup_params text[], client_hook_arg text)
+RETURNS text[]
+LANGUAGE plpgsql AS $$
+DECLARE
+ elem text;
+ paramname text;
+ paramvalue text;
+BEGIN
+ FOREACH elem IN ARRAY startup_params
+ LOOP
+ IF elem IS NULL THEN
+ RAISE EXCEPTION 'Startup params may not be null';
+ END IF;
+
+ IF paramname IS NULL THEN
+ paramname := elem;
+ ELSIF paramvalue IS NULL THEN
+ paramvalue := elem;
+ ELSE
+ RAISE NOTICE 'got param: % = %', paramname, paramvalue;
+ paramname := NULL;
+ paramvalue := NULL;
+ END IF;
+ END LOOP;
+
+ RETURN ARRAY['pglo_plhooks_demo_startup_ran', 'true', 'otherparam', '42'];
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_txn_filter(origin_id int, client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- Not much to filter on really...
+ RAISE NOTICE 'Got tx with origin %',origin_id;
+ RETURN true;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_row_filter(affected_rel regclass, change_type "char", client_hook_arg text)
+RETURNS boolean
+LANGUAGE plpgsql AS $$
+BEGIN
+ -- This is a totally absurd test, since it checks if the upstream user
+ -- doing replication has rights to make modifications that have already
+ -- been committed and are being decoded for replication. Still, it shows
+ -- how the hook works...
+ IF pg_catalog.has_table_privilege(current_user, affected_rel,
+ CASE change_type WHEN 'I' THEN 'INSERT' WHEN 'U' THEN 'UPDATE' WHEN 'D' THEN 'DELETE' END)
+ THEN
+ RETURN true;
+ ELSE
+ RETURN false;
+ END IF;
+END;
+$$;
+
+CREATE FUNCTION pglo_plhooks_demo_shutdown(client_hook_arg text)
+RETURNS void
+LANGUAGE plpgsql AS $$
+BEGIN
+ RAISE NOTICE 'Decoding shutdown';
+END;
+$$
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
new file mode 100644
index 0000000..a5144f0
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.c
@@ -0,0 +1,414 @@
+#include "postgres.h"
+
+#include "pglogical_output/hooks.h"
+
+#include "access/xact.h"
+
+#include "catalog/pg_type.h"
+
+#include "nodes/makefuncs.h"
+
+#include "parser/parse_func.h"
+
+#include "replication/reorderbuffer.h"
+
+#include "utils/acl.h"
+#include "utils/array.h"
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
+#include "utils/rel.h"
+
+#include "fmgr.h"
+#include "miscadmin.h"
+
+PG_MODULE_MAGIC;
+
+PGDLLEXPORT extern Datum pglo_plhooks_setup_fn(PG_FUNCTION_ARGS);
+PG_FUNCTION_INFO_V1(pglo_plhooks_setup_fn);
+
+void pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args);
+void pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args);
+bool pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args);
+bool pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args);
+
+typedef struct PLHPrivate
+{
+ const char *client_arg;
+ Oid startup_hook;
+ Oid shutdown_hook;
+ Oid row_filter_hook;
+ Oid txn_filter_hook;
+ MemoryContext hook_call_context;
+} PLHPrivate;
+
+static void read_parameters(PLHPrivate *private, List *in_params);
+static Oid find_startup_hook(const char *proname);
+static Oid find_shutdown_hook(const char *proname);
+static Oid find_row_filter_hook(const char *proname);
+static Oid find_txn_filter_hook(const char *proname);
+static void exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params);
+
+void
+pglo_plhooks_startup(struct PGLogicalStartupHookArgs *startup_args)
+{
+ PLHPrivate *private;
+
+ /* pglogical_output promises to call us in a tx */
+ Assert(IsTransactionState());
+
+ /* Allocated in hook memory context, scoped to the logical decoding session: */
+ startup_args->private_data = private = (PLHPrivate*)palloc(sizeof(PLHPrivate));
+
+ private->startup_hook = InvalidOid;
+ private->shutdown_hook = InvalidOid;
+ private->row_filter_hook = InvalidOid;
+ private->txn_filter_hook = InvalidOid;
+ /* client_arg is the empty string when not specified to simplify function calls */
+ private->client_arg = "";
+
+ read_parameters(private, startup_args->in_params);
+
+ private->hook_call_context = AllocSetContextCreate(CurrentMemoryContext,
+ "pglogical_output plhooks hook call context",
+ ALLOCSET_SMALL_MINSIZE,
+ ALLOCSET_SMALL_INITSIZE,
+ ALLOCSET_SMALL_MAXSIZE);
+
+
+ if (private->startup_hook != InvalidOid)
+ exec_user_startup_hook(private, startup_args->in_params, &startup_args->out_params);
+}
+
+void
+pglo_plhooks_shutdown(struct PGLogicalShutdownHookArgs *shutdown_args)
+{
+ PLHPrivate *private = (PLHPrivate*)shutdown_args->private_data;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->shutdown_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo shutdown hook with %s", private->client_arg);
+ (void) OidFunctionCall1(
+ private->shutdown_hook,
+ CStringGetTextDatum(private->client_arg));
+ elog(DEBUG3, "called pglo shutdown hook");
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+}
+
+bool
+pglo_plhooks_row_filter(struct PGLogicalRowFilterArgs *rowfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)rowfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+ if (OidIsValid(private->row_filter_hook))
+ {
+ char change_type;
+ switch (rowfilter_args->change_type)
+ {
+ case REORDER_BUFFER_CHANGE_INSERT:
+ change_type = 'I';
+ break;
+ case REORDER_BUFFER_CHANGE_UPDATE:
+ change_type = 'U';
+ break;
+ case REORDER_BUFFER_CHANGE_DELETE:
+ change_type = 'D';
+ break;
+ default:
+ elog(ERROR, "unknown change type %d", rowfilter_args->change_type);
+ change_type = '0'; /* silence compiler */
+ }
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+ elog(DEBUG3, "calling pglo row filter hook with (%u,%c,%s)",
+ rowfilter_args->changed_rel->rd_id, change_type,
+ private->client_arg);
+ ret = DatumGetBool(OidFunctionCall3(
+ private->row_filter_hook,
+ ObjectIdGetDatum(rowfilter_args->changed_rel->rd_id),
+ CharGetDatum(change_type),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "called pglo row filter hook, returns %d", (int)ret);
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+bool
+pglo_plhooks_txn_filter(struct PGLogicalTxnFilterArgs *txnfilter_args)
+{
+ PLHPrivate *private = (PLHPrivate*)txnfilter_args->private_data;
+ bool ret = true;
+ MemoryContext old_ctx;
+
+ Assert(private != NULL);
+
+
+ if (OidIsValid(private->txn_filter_hook))
+ {
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ elog(DEBUG3, "calling pglo txn filter hook with (%hu,%s)",
+ txnfilter_args->origin_id, private->client_arg);
+ ret = DatumGetBool(OidFunctionCall2(
+ private->txn_filter_hook,
+ UInt16GetDatum(txnfilter_args->origin_id),
+ CStringGetTextDatum(private->client_arg)));
+ elog(DEBUG3, "calling pglo txn filter hook, returns %d", (int)ret);
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+ }
+
+ return ret;
+}
+
+Datum
+pglo_plhooks_setup_fn(PG_FUNCTION_ARGS)
+{
+ struct PGLogicalHooks *hooks = (struct PGLogicalHooks*) PG_GETARG_POINTER(0);
+
+ /* Your code doesn't need this, it's just for the tests: */
+ Assert(hooks != NULL);
+ Assert(hooks->hooks_private_data == NULL);
+ Assert(hooks->startup_hook == NULL);
+ Assert(hooks->shutdown_hook == NULL);
+ Assert(hooks->row_filter_hook == NULL);
+ Assert(hooks->txn_filter_hook == NULL);
+
+ /*
+ * Just assign the hook pointers. We're not meant to do much
+ * work here.
+ *
+ * Note that private_data is left untouched, to be set up by the
+ * startup hook.
+ */
+ hooks->startup_hook = pglo_plhooks_startup;
+ hooks->shutdown_hook = pglo_plhooks_shutdown;
+ hooks->row_filter_hook = pglo_plhooks_row_filter;
+ hooks->txn_filter_hook = pglo_plhooks_txn_filter;
+ elog(DEBUG3, "configured pglo hooks");
+
+ PG_RETURN_VOID();
+}
+
+static void
+exec_user_startup_hook(PLHPrivate *private, List *in_params, List **out_params)
+{
+ ArrayType *startup_params;
+ Datum ret;
+ ListCell *lc;
+ Datum *startup_params_elems;
+ bool *startup_params_isnulls;
+ int n_startup_params;
+ int i;
+ MemoryContext old_ctx;
+
+
+ old_ctx = MemoryContextSwitchTo(private->hook_call_context);
+
+ /*
+ * Build the input parameter array. NULL parameters are passed as the
+ * empty string for the sake of convenience. Each param is two
+ * elements, a key then a value element.
+ */
+ n_startup_params = list_length(in_params) * 2;
+ startup_params_elems = (Datum*)palloc0(sizeof(Datum)*n_startup_params);
+
+ i = 0;
+ foreach (lc, in_params)
+ {
+ DefElem * elem = (DefElem*)lfirst(lc);
+ const char *val;
+
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ val = "";
+ else
+ val = strVal(elem->arg);
+
+ startup_params_elems[i++] = CStringGetTextDatum(elem->defname);
+ startup_params_elems[i++] = CStringGetTextDatum(val);
+ }
+ Assert(i == n_startup_params);
+
+ startup_params = construct_array(startup_params_elems, n_startup_params,
+ TEXTOID, -1, false, 'i');
+
+ ret = OidFunctionCall2(
+ private->startup_hook,
+ PointerGetDatum(startup_params),
+ CStringGetTextDatum(private->client_arg));
+
+ /*
+ * deconstruct return array and add pairs of results to a DefElem list.
+ */
+ deconstruct_array(DatumGetArrayTypeP(ret), TEXTARRAYOID,
+ -1, false, 'i', &startup_params_elems, &startup_params_isnulls,
+ &n_startup_params);
+
+
+ *out_params = NIL;
+ for (i = 0; i < n_startup_params; i = i + 2)
+ {
+ char *value;
+ DefElem *elem;
+
+ if (startup_params_isnulls[i])
+ elog(ERROR, "Array entry corresponding to a key was null at idx=%d", i);
+
+ if (startup_params_isnulls[i+1])
+ value = "";
+ else
+ value = TextDatumGetCString(startup_params_elems[i+1]);
+
+ elem = makeDefElem(
+ TextDatumGetCString(startup_params_elems[i]),
+ (Node*)makeString(value));
+
+ *out_params = lcons(elem, *out_params);
+ }
+
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextReset(private->hook_call_context);
+}
+
+static void
+read_parameters(PLHPrivate *private, List *in_params)
+{
+ ListCell *option;
+
+ foreach(option, in_params)
+ {
+ DefElem *elem = lfirst(option);
+
+ if (pg_strcasecmp("pglo_plhooks.client_hook_arg", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.client_hook_arg may not be NULL");
+ private->client_arg = pstrdup(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.startup_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.startup_hook may not be NULL");
+ private->startup_hook = find_startup_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.shutdown_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.shutdown_hook may not be NULL");
+ private->shutdown_hook = find_shutdown_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.txn_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.txn_filter_hook may not be NULL");
+ private->txn_filter_hook = find_txn_filter_hook(strVal(elem->arg));
+ }
+
+ if (pg_strcasecmp("pglo_plhooks.row_filter_hook", elem->defname) == 0)
+ {
+ if (elem->arg == NULL || strVal(elem->arg) == NULL)
+ elog(ERROR, "pglo_plhooks.row_filter_hook may not be NULL");
+ private->row_filter_hook = find_row_filter_hook(strVal(elem->arg));
+ }
+ }
+}
+
+static Oid
+find_hook_fn(const char *funcname, Oid funcargtypes[], int nfuncargtypes, Oid returntype)
+{
+ Oid funcid;
+ List *qname;
+
+ qname = stringToQualifiedNameList(funcname);
+
+ /* find the the function */
+ funcid = LookupFuncName(qname, nfuncargtypes, funcargtypes, false);
+
+ /* Check expected return type */
+ if (get_func_rettype(funcid) != returntype)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_WRONG_OBJECT_TYPE),
+ errmsg("function %s doesn't return expected type %d",
+ NameListToString(qname), returntype)));
+ }
+
+ if (pg_proc_aclcheck(funcid, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK)
+ {
+ const char * username;
+#if PG_VERSION_NUM >= 90500
+ username = GetUserNameFromId(GetUserId(), false);
+#else
+ username = GetUserNameFromId(GetUserId());
+#endif
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ errmsg("current user %s does not have permission to call function %s",
+ username, NameListToString(qname))));
+ }
+
+ list_free_deep(qname);
+
+ return funcid;
+}
+
+static Oid
+find_startup_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = TEXTARRAYOID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, VOIDOID);
+}
+
+static Oid
+find_shutdown_hook(const char *proname)
+{
+ Oid argtypes[1];
+
+ argtypes[0] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 1, VOIDOID);
+}
+
+static Oid
+find_row_filter_hook(const char *proname)
+{
+ Oid argtypes[3];
+
+ argtypes[0] = REGCLASSOID;
+ argtypes[1] = CHAROID;
+ argtypes[2] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 3, BOOLOID);
+}
+
+static Oid
+find_txn_filter_hook(const char *proname)
+{
+ Oid argtypes[2];
+
+ argtypes[0] = INT4OID;
+ argtypes[1] = TEXTOID;
+
+ return find_hook_fn(proname, argtypes, 2, BOOLOID);
+}
diff --git a/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
new file mode 100644
index 0000000..647b9ef
--- /dev/null
+++ b/contrib/pglogical_output_plhooks/pglogical_output_plhooks.control
@@ -0,0 +1,4 @@
+comment = 'pglogical_output pl hooks'
+default_version = '1.0'
+module_pathname = '$libdir/pglogical_output_plhooks'
+relocatable = false
--
2.1.0
I didn't receive a response on the bugs mailing list for the following bug,
so I was hoping we could triage to someone with more familiarity with
Postgres internals than I to fix.
This ticket seems like folks who are invested in logical decoding.
The attached script is a simple workload that logical decoding is unable to
decode. It causes an unrecoverable crash in the logical decoder with
'ERROR: subxact logged without previous toplevel record'.
On Thu, Jan 7, 2016 at 12:44 AM, Craig Ringer <craig@2ndquadrant.com> wrote:
Show quoted text
Here's v5 of the pglogical patch.
Changes:
* Implement relation metadata caching
* Add the relmeta_cache_size parameter for cache control
* Add an extension to get version information
* Create the pglogical_output header directory on install
* Restore 9.4 compatibility (it's small)
* Allow row filter hooks to see details of the changed tuple
* Remove forward_changesets from pglogical_output (use a hook if you want
this functionality)I'm not sure if 9.4 compat will be desirable or not. It's handy to avoid
needing a separate backported version, but also confusing to do a PGXS
build within a 9.6 tree against 9.4...--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Attachments:
On 2016-01-07 09:28:29 -0800, Jarred Ward wrote:
I didn't receive a response on the bugs mailing list for the following bug,
so I was hoping we could triage to someone with more familiarity with
Postgres internals than I to fix.
Please don't post to unrelated threads, that just confuses things.
Andres
PS: Yes, I do plan to look at that issue at some point.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
W dniu 07.01.2016, czw o godzinie 15∶50 +0800, użytkownik Craig Ringer
napisał:
On 7 January 2016 at 01:17, Peter Eisentraut <peter_e@gmx.net> wrote:
On 12/22/15 4:55 AM, Craig Ringer wrote:
I'm a touch frustrated by that, as a large part of the point of
submitting the output plugin separately and in advance of thedownstream
was to get attention for it separately, as its own entity. A lot
of
effort has been put into making this usable for more than just a
data
source for pglogical's replication tools.
Maybe chosen name was not the best one - I assumed from the very
eginning that it's replication solution and not something separate.
I can't imagine that there is a lot of interest in a replication
tool
where you only get one side of it, no matter how well-designed or
general it is.Well, the other part was posted most of a week ago.
/messages/by-id/5685BB86.5010901@2ndquadrant.com
... but this isn't just about replication. At least, not just to
another PostgreSQL instance. This plugin is designed to be general
enough to use for replication to other DBMSes (via appropriate
receivers), to replace trigger-based data collection in existing
replication systems, for use in audit data collection, etc.Want to get a stream of data out of PostgreSQL in a consistent,
simple way, without having to add triggers or otherwise interfere
with the origin database? That's the purpose of this plugin, and it
doesn't care in the slightest what the receiver wants to do with that
data. It's been designed to be usable separately from pglogical
downstream and - before the Python tests were rejected in discussions
on this list - was tested using a test suite completely separate to
the pglogical downstream using psycopg2 to make sure no unintended
interdependencies got introduced.You can do way more than that with the output plugin but you have to
write your own downstream/receiver for the desired purpose, since
using a downstream based on bgworkers and SPI won't make any sense
outside PostgreSQL.
Put those 3 paragraphs into README.md - and this is not a joke.
This is very good rationale behind this plugin; for now README
starts with link to documentation describing logical decoding
and the second paragraph talks about replication.
So when replication (and only it) is in README, it should be
no wonder that people (only - or mostly) think about replication.
Maybe we should think about changing the name to something like
logical_decoder or logical_streamer, to divorce this plugin
from pglogical? Currently even name suggests tight coupling - and
in other way than it should be. pglogical depends on this plugin,
not the other way around.
If you just want a canned product to use, see the pglogical post
above for the downstream code.
Ultimately, what people will want to do with this is
replicate things, not muse about its design aspects. So if we're
going
to ship a replication solution in PostgreSQL core, we should ship
all
the pieces that make the whole system work.I don't buy that argument. Doesn't that mean logical decoding
shouldn't have been accepted? Or the initial patches for parallel
query? Or any number of other things that're part of incremental
development solutions?(This also seems to contradict what you then argue below, that the
proposed feature is too broad and does too much.)I'd be happy to see both parts go in, but I'm frustrated that
nobody's willing to see beyond "replicate from one Pg to another Pg"
and see all the other things you can do. Want to replicate to Oracle
/ MS-SQL / etc? This will help a lot and solve a significant part of
the problem for you. Want to stream data to append-only audit logs?
Ditto. But nope, it's all about PostgreSQL to PostgreSQL.Please try to look further into what client applications can do with
this directly. I already know it meets the needs of the pglogical
downstream. What I was hoping to achieve with posting the output
plugin earlier was to get some thought going about what *else* it'd
be good for.Again: pglogical is posted now (it just took longer than expected to
get ready) and I'll be happy to see both it and the output plugin
included. I just urge people to look at the output plugin as more
than a tightly coupled component of pglogical.Maybe some quality name bikeshedding for the output plugin would help
;)Also, I think there are two kinds of general systems: common core,
and
all possible features. A common core approach could probably be
made
acceptable with the argument that anyone will probably want to do
things
this way, so we might as well implement it once and give it to
people.That's what we're going for here. Extensible, something people can
build on and use.
In a way, the logical decoding interface is the common core, as we
currently understand it. But this submission clearly has a lot of
features beyond just the basicsReally? What would you cut? What's beyond the basics here? What
basics are you thinking of, i.e what set of requirements are you
working towards / needs are you seeking to meet?We cut this to the bone to produce a minimum viable logical
replication solution. Especially the output plugin.Cut the hook interfaces for row and xact filtering? You lose the
ability to use replication origins, crippling functionality, and for
no real gain in simplicity.Remove JSON support? That's what most people are actually likely to
want to use when using the output plugin directly, and it's important
for debugging/tracing/diagnostics. It's a separate feature, to be
sure, but it's also a pretty trivial addition.
and we could probably go through them
one by one and ask, why do we need this bit? So that kind of
system
will be very hard to review as a standalone submission.Again, I disagree. I think you're looking at this way too narrowly.
I find it quite funny, actually. Here we go and produce something
that's a nice re-usable component that other people can use in their
products and solutions ... and all anyone does is complain that the
other part required to use it as a canned product isn't posted yet
(though it is now). But with BDR all anyone ever does is complain
that it's too tightly coupled to the needs of a single product and
the features extracted from it, like replication origins, should be
more generic and general purpose so other people can use them in
their products too. Which is it going to be?It would be helpful if you could take a step back and describe what
*you* think logical replication for PostgreSQL should look like. You
clearly have a picture in mind of what it should be, what
requirements it satisfies, etc. If you're going to argue based on
that it'd be very helpful to describe it. I might've missed some
important points you've seen and you might've overlooked issues I've
seen.
This is rather long, but I do not want to cut to much, because
it shows slight problem with workflow in PostgreSQL community.
I'm writing as someone trying to increase my involvement,
not fully outsider, but not yet feeling fully belonging.
I'll try to explain what I mean taking this patch as example.
It started
as part of pglogical replication, but you split
it to ease review. But
this origin shows - in name, in comments,
in README. It's good example
of scratching itch - but without
connection to others, or maybe without
wider picture.
Don't get me wrong - having code to discuss is much
better
than just bikeshedding what we'd like to have.
And changes in v5
(like caching and passing tuples to hooks)
give hope for some work.
OTOH,
by looking at parallel queries, maybe once it lands
in repository it'll
get more attention?
I can feel your frustration. Coming to community without own
itch to scratch is also a bit frustrating - I do not know where
to start, what needs the most attention. I can see that
commitfests are in dire need for reviewers, so I started with
them. But at the same time I can only check whether code looks
correct, applies cleanly, whether it compiles, whether
tests pass.
I do not see bigger picture - and also cannot see emails with
discussion about long- or mid-term direction or vision.
It makes harder to feel that it matters and to decide
which patch look at.
Both communities I feel attached to (Debian and PostgreSQL)
differ from many highly visible FLOSS projects that they
do not have one backing company, nor benevolent dictator.
It gives them freedom to pursue different goals without
risk of disrupting power structure, but at the same time
it make it harder to connect the dots and see how project
is doing.
OK, we went quite far away from review. I do not have closing
remarks - only that I hope to provide better review by the weekend.
And let's discuss name - I do not fully like pglogical_decoding.
--
Tomasz Rybak GPG/PGP key ID: 2AD5 9860
Fingerprint A481 824E 7DD3 9C0E C40A 488E C654 FB33 2AD5 9860
http://member.acm.org/~tomaszrybak
I just quickly went through patch v5.
It's rather big patch, on (or beyond) my knowledge of PostgreSQL to perform high-quality review. But during this week I'll try to send reviews of parts of the code, as going through it in one sitting seems impossible.
One proposed change - update copyright to 2016.
i'd also propose to change of pglogical_output_control.in:
comment = 'general purpose logical decoding plugin'
to something like "general-purpoer plugin decoding and generating stream of logical changes"
We might also think about changing name of plugin to something resembling "logical_streaming_decoder" or even "logical_streamer"
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 19 January 2016 at 05:47, Tomasz Rybak <tomasz.rybak@post.pl> wrote:
I just quickly went through patch v5.
It's rather big patch, on (or beyond) my knowledge of PostgreSQL to
perform high-quality review. But during this week I'll try to send reviews
of parts of the code, as going through it in one sitting seems impossible.One proposed change - update copyright to 2016.
i'd also propose to change of pglogical_output_control.in:
comment = 'general purpose logical decoding plugin'
to something like "general-purpoer plugin decoding and generating stream
of logical changes"We might also think about changing name of plugin to something resembling
"logical_streaming_decoder" or even "logical_streamer"
I'm open to ideas there but I'd want some degree of consensus before
undertaking the changes required.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
The following review has been posted through the commitfest application:
make installcheck-world: not tested
Implements feature: not tested
Spec compliant: not tested
Documentation: not tested
First part of code review (for about 1/3rd of code):
pglogical_output.h:
+ /* Protocol capabilities */
+ #define PGLOGICAL_PROTO_VERSION_NUM 1
+ #define PGLOGICAL_PROTO_MIN_VERSION_NUM 1
Is this protocol version number and minimal recognized version number,
or major and minor version number? I assume that PGLOGICAL_PROTO_VERSION_NUM
is current protocol version (as in config max_proto_version and
min_proto_version). But why we have MIN_VERSION and not MAX_VERSION?
From code in pglogical_output.c (lines 215-225 it looks like
PGLOGICAL_PROTO_VERSION_NUM is maximum, and PGLOGICAL_PROTO_MIN_VERSION_NUM
is treated as minimal protocol version number.
I can see that those constants are exported in pglogical_infofuncs.c and
pglogical.sql, but I do not understand omission of MAX.
+ typedef struct HookFuncName
+ typedef struct PGLogicalTupleData
I haven't found those used anything, and they are not mentioned in
documentation. Are those structures needed?
+ pglogical_config.c:
+ switch(get_param_key(elem->defname))
+ {
+ val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
Why do we need this line here? All cases contain some variant of
val = get_param_value(elem, false, TYPE);
as first line after "case PARAM_*:" so this line should be removed.
+ val = get_param(options, "startup_params_format", false, false,
+ OUTPUT_PARAM_TYPE_UINT32, &found);
+
+ params_format = DatumGetUInt32(val);
Shouldn't we check "found" here? We work with val (which is Datum(0)) - won't
it throw SIGFAULT, or similar?
Additionally - I haven't found any case where code would use "found"
passed from get_param() - so as it's not used it might be removed.
pglogical_output.c:
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(Datum) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeofint != 0
+ && data->client_binary_sizeofint != sizeof(int))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(int) mismatch");
+ return false;
+ }
+
+ if (data->client_binary_sizeoflong != 0
+ && data->client_binary_sizeoflong != sizeof(long))
+ {
+ elog(DEBUG1, "Binary mode rejected: Server and client endian sizeof(long) mismatch");
Isn't "endian" here case of copy-paste from first error?
Error messages should rather look like:
elog(DEBUG1, "Binary mode rejected: Server and client sizeof(Datum) mismatch");
+ static void pg_decode_shutdown(LogicalDecodingContext * ctx)
In pg_decode_startup we create main memory context, and create hooks memory
context. In pg_decode_shutdown we delete hooks memory context but not main
context. Is this OK, or should we also add:
MemoryContextDelete(data->context);
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 20 January 2016 at 06:23, Tomasz Rybak <tomasz.rybak@post.pl> wrote:
The following review has been posted through the commitfest application:
Thanks!
+ /* Protocol capabilities */ + #define PGLOGICAL_PROTO_VERSION_NUM 1 + #define PGLOGICAL_PROTO_MIN_VERSION_NUM 1 Is this protocol version number and minimal recognized version number, or major and minor version number? I assume that PGLOGICAL_PROTO_VERSION_NUM is current protocol version (as in config max_proto_version and min_proto_version). But why we have MIN_VERSION and not MAX_VERSION?From code in pglogical_output.c (lines 215-225 it looks like
PGLOGICAL_PROTO_VERSION_NUM is maximum, and PGLOGICAL_PROTO_MIN_VERSION_NUM
is treated as minimal protocol version number.I can see that those constants are exported in pglogical_infofuncs.c and
pglogical.sql, but I do not understand omission of MAX.
Thanks for stopping to think about this. It's one of the areas I really
want to get right but I'm not totally confident of.
The idea here is that we want downwards compatibility as far as possible
and maintainable but we can't really be upwards compatible for breaking
protocol revisions. So the output plugin's native protocol version is
inherently the max protocol version and we don't need a separate MAX.
The downstream connects and declares to the upstream "I speak protocol 2
through 3". The upstream sees this and replies "I speak protocol 1 through
2. We have protocol 2 in common so I will use that." Or alternately replies
with an error "I only speak protocol 1 so we have no protocol in common".
This is done via the initial parameters passed by the downstream to the
logical decoding plugin and then via the startup reply message that's the
first message on the logical decoding stream.
We can't do a better handshake than this because the underlying walsender
protocol and output plugin API only gives us one chance to send free-form
information to the output plugin and it has to do so before the output
plugin can send anything to the downstream.
As much as possible I want to avoid ever needing to do a protocol bump
anyway, since it'll involve adding conditionals and duplication. That's why
the protocol tries so hard to be extensible and rely on declared
capabilities rather than protocol version bumps. But I'd rather plan for it
than be unable to ever do it in future.
Much (all?) of this is discussed in the protocol docs. I should probably
double check that and add a comment that refers to them there.
+ typedef struct HookFuncName
Thanks. That's residue from the prior implementation of hooks, which used
direct pg_proc lookups and cached the FmgrInfo in a dynahash. It's no
longer required now that we're using a single hook entry point and direct C
function calls. Dead code, removed.
+ typedef struct PGLogicalTupleData
I haven't found those used anything, and they are not mentioned in
documentation. Are those structures needed?
That snuck in from the pglogical downstream during the split into a
separate tree. It's dead code as far as pglogical_output is concerned.
Removed.
+ pglogical_config.c: + switch(get_param_key(elem->defname)) + { + val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32);
Why do we need this line here? All cases contain some variant of
val = get_param_value(elem, false, TYPE);
as first line after "case PARAM_*:" so this line should be removed.
Correct. That seems to be an escapee editing error. Thanks, removed.
+ val = get_param(options, "startup_params_format", false, false, + OUTPUT_PARAM_TYPE_UINT32, &found); + + params_format = DatumGetUInt32(val); Shouldn't we check "found" here? We work with val (which is Datum(0)) - won't it throw SIGFAULT, or similar?
get_param is called with missing_ok=false so it ERRORs and can never return
!found . In any case it'd return (Datum)0 so we'd just get (uint32)0 not a
crash.
To make this clearer I've changed get_param so it supports NULL as a value
for found.
Additionally - I haven't found any case where code would use "found"
passed from get_param() - so as it's not used it might be removed.
Probably, but I expect it to be useful later. It was used before a
restructure of how params get read. I don't mind removing it if you feel
strongly about it, but it'll probably just land up being added back at some
point.
+ elog(DEBUG1, "Binary mode rejected: Server and client
endian sizeof(long) mismatch");
Isn't "endian" here case of copy-paste from first error?
Yes, it is. Thanks.
+ static void pg_decode_shutdown(LogicalDecodingContext * ctx)
In pg_decode_startup we create main memory context, and create hooks memory
context. In pg_decode_shutdown we delete hooks memory context but not main
context. Is this OK, or should we also add:
MemoryContextDelete(data->context);
<http://www.postgresql.org/mailpref/pgsql-hackers>
Good catch. I think a better fix is to make it a child of the logical
decoding context so it's deleted automatically. It's actually unnecessary
to delete data->hooks_mctxt here for the same reason.
Amended.
I've also patched the tests to handle the failure to fail on an
incorrect startup_params_format .
Changes pushed to
https://github.com/2ndquadrant/postgres/tree/dev/pglogical-output
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
W dniu 20.01.2016, śro o godzinie 13∶54 +0800, użytkownik Craig Ringer
napisał:
On 20 January 2016 at 06:23, Tomasz Rybak <tomasz.rybak@post.pl>
wrote:The following review has been posted through the commitfest
application:Thanks!
+ /* Protocol capabilities */ + #define PGLOGICAL_PROTO_VERSION_NUM 1 + #define PGLOGICAL_PROTO_MIN_VERSION_NUM 1 Is this protocol version number and minimal recognized version number, or major and minor version number? I assume that PGLOGICAL_PROTO_VERSION_NUM is current protocol version (as in config max_proto_version and min_proto_version). But why we have MIN_VERSION and not MAX_VERSION?From code in pglogical_output.c (lines 215-225 it looks like
PGLOGICAL_PROTO_VERSION_NUM is maximum, and
PGLOGICAL_PROTO_MIN_VERSION_NUM
is treated as minimal protocol version number.I can see that those constants are exported in
pglogical_infofuncs.c and
pglogical.sql, but I do not understand omission of MAX.Thanks for stopping to think about this. It's one of the areas I
really want to get right but I'm not totally confident of.The idea here is that we want downwards compatibility as far as
possible and maintainable but we can't really be upwards compatible
for breaking protocol revisions. So the output plugin's native
protocol version is inherently the max protocol version and we don't
need a separate MAX.The downstream connects and declares to the upstream "I speak
protocol 2 through 3". The upstream sees this and replies "I speak
protocol 1 through 2. We have protocol 2 in common so I will use
that." Or alternately replies with an error "I only speak protocol 1
so we have no protocol in common". This is done via the initial
parameters passed by the downstream to the logical decoding plugin
and then via the startup reply message that's the first message on
the logical decoding stream.We can't do a better handshake than this because the underlying
walsender protocol and output plugin API only gives us one chance to
send free-form information to the output plugin and it has to do so
before the output plugin can send anything to the downstream.As much as possible I want to avoid ever needing to do a protocol
bump anyway, since it'll involve adding conditionals and duplication.
That's why the protocol tries so hard to be extensible and rely on
declared capabilities rather than protocol version bumps. But I'd
rather plan for it than be unable to ever do it in future.Much (all?) of this is discussed in the protocol docs. I should
probably double check that and add a comment that refers to them
there.
Thanks for explanation. I'll think about it more and try to propose
something for this.
Best regards.
--
Tomasz Rybak GPG/PGP key ID: 2AD5 9860
Fingerprint A481 824E 7DD3 9C0E C40A 488E C654 FB33 2AD5 9860
http://member.acm.org/~tomaszrybak
The following review has been posted through the commitfest application:
make installcheck-world: not tested
Implements feature: not tested
Spec compliant: not tested
Documentation: not tested
I revied more files:
pglogical_proto_native.c
+ pq_sendbyte(out, 'N'); /* column name block follows */
+ attname = NameStr(att->attname);
+ len = strlen(attname) + 1;
+ pq_sendint(out, len, 2);
+ pq_sendbytes(out, attname, len); /* data */
Identifier names are limited to 63 - so why we're sending 2 bytes here?
I do not have hard feelings about this - just curiousity. For:
+ pq_sendbyte(out, nspnamelen); /* schema name length */
+ pq_sendbytes(out, nspname, nspnamelen);
+ pq_sendbyte(out, relnamelen); /* table name length */
+ pq_sendbytes(out, relname, relnamelen);
schema and relation name we send 1 byte, for attribute 2. Strange, bit inconsistent.
+ pq_sendbyte(out, 'B'); /* BEGIN */
+
+ /* send the flags field its self */
+ pq_sendbyte(out, flags);
Comment: "flags field its self"? Shouldn't be "... itself"?
Similarly in write_origin; write_insert just says:
+ /* send the flags field */
+ pq_sendbyte(out, flags);
Speaking about flags - in most cases they are 0; only for attributes
we might have:
+ flags |= IS_REPLICA_IDENTITY;
I assume that flags field is put into protocol for future needs?
+ /* FIXME support whole tuple (O tuple type) */
+ if (oldtuple != NULL)
+ {
+ pq_sendbyte(out, 'K'); /* old key follows */
+ pglogical_write_tuple(out, data, rel, oldtuple);
+ }
I don't fully understand. We are sending whole old tuple here,
so this FIXME should be more about supporting sending just keys.
But then comment "old key follows" is not true. Or am I missing
something here?
Similarly for write_delete.
+ pq_sendbyte(out, 'S'); /* message type field */
+ pq_sendbyte(out, 1); /* startup message version */
For now protocol is 1, but for code readability it might be better
to change this line to:
+ pq_sendbyte(out, PGLOGICAL_PROTO_VERSION_NUM); /* startup message version */
Just for the sake of avoiding code repetition:
+ for (i = 0; i < desc->natts; i++)
+ {
+ if (desc->attrs[i]->attisdropped)
+ continue;
+ nliveatts++;
+ }
+ pq_sendint(out, nliveatts, 2);
The exact same code is in write_tuple and write_attrs. I don't know what's
policy for refactoring, but this might be extracted into separate function.
+ else if (att->attlen == -1)
+ {
+ char *data = DatumGetPointer(values[i]);
+
+ /* send indirect datums inline */
+ if (VARATT_IS_EXTERNAL_INDIRECT(values[i]))
+ {
+ struct varatt_indirect redirect;
+ VARATT_EXTERNAL_GET_POINTER(redirect, data);
+ data = (char *) redirect.pointer;
+ }
I really don't like this. We have function parameter "data" and now
are creating new variable with the same name. It might lead to confusion
and some long debugging sessions. Please change the name of this variable.
Maybe attr_data would be OK? Or outputbytes, like below, for case 'b' and default?
pglogical_proto_json.c
+ appendStringInfo(out, ", \"origin_lsn\":\"%X/%X\"",
+ (uint32)(txn->origin_lsn >> 32), (uint32)(txn->origin_lsn));
I remember there was discussion on *-hackers recently about %X/%X; I'll
try to find it and check whether it's according to final conclusion.
pglogical_relmetacache.c
First. In pglogical_output.c, in pg_decode_startup we are calling
init_relmetacache. I haven't found call to destroy_relmetacache
and comment says that it must be called at backend shutdown.
Is it guaranteed? Or will cache get freed with its context?
+ /* Find cached function info, creating if not found */
+ hentry = (struct PGLRelMetaCacheEntry*) hash_search(RelMetaCache,
+ (void *)(&RelationGetRelid(rel)),
+ HASH_ENTER, &found);
+
+ if (!found)
+ {
+ Assert(hentry->relid = RelationGetRelid(rel));
+ hentry->is_cached = false;
+ hentry->api_private = NULL;
+ }
+
+ Assert(hentry != NULL);
Shouldn't Assert be just after calling hash_search? We're (if !found)
dereferencing hentry and only after checking whether it's not NULL.
I haven't found relevance of relmeta_cache_size attribute.
It's set to value coming from client - but then the only thing that matters
is whether it is 0 or not. I'll try to reread DESIGN.md session about cache,
but for now (quite late and I'm quite tired) it is not clear on the first reading.
Best regards.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 21 January 2016 at 06:23, Tomasz Rybak <tomasz.rybak@post.pl> wrote:
I reviewed more files:
Thanks.
Can you try to put more whitespace between items? It can be hard to follow
at the moment.
pglogical_proto_native.c
+ pq_sendbyte(out, 'N'); /* column name block follows */ + attname = NameStr(att->attname); + len = strlen(attname) + 1; + pq_sendint(out, len, 2); + pq_sendbytes(out, attname, len); /* data */ Identifier names are limited to 63 - so why we're sending 2 bytes here?
Good question. It should be one byte. I'll need to amend the protocol for
that, but I don't think that's a problem this early on.
+ pq_sendbyte(out, 'B'); /* BEGIN */ + + /* send the flags field its self */ + pq_sendbyte(out, flags); Comment: "flags field its self"? Shouldn't be "... itself"? Similarly in write_origin; write_insert just says:
itself is an abbreviation of its self.
+ /* send the flags field */
+ pq_sendbyte(out, flags);Speaking about flags - in most cases they are 0; only for attributes
we might have:
+ flags |= IS_REPLICA_IDENTITY;
I assume that flags field is put into protocol for future needs?
Correct. The protocol specifies (in most places; need to double check all
sites) that the lower 4 bits are reserved and must be treated as an ERROR
if set. The high 4 bits must be ignored if set and not recognised. That
gives us some room to wiggle without bumping the protocol version
incompatibly, and lets us use capabilities (via client-supplied parameters)
to add extra information on the wire.
+ /* FIXME support whole tuple (O tuple type) */
+ if (oldtuple != NULL) + { + pq_sendbyte(out, 'K'); /* old key follows */ + pglogical_write_tuple(out, data, rel, oldtuple); + } I don't fully understand. We are sending whole old tuple here, so this FIXME should be more about supporting sending just keys. But then comment "old key follows" is not true. Or am I missing something here?
In wal_level=logical the tuple that's written is an abbreviated tuple
containing data only for the REPLICA IDENTITY fields.
Ideally we'd also be able to support sending the _whole_ old tuple, but
this would require the ability to log the whole old tuple in WAL when
logging a DELETE or UPDATE into WAL. This isn't so much a FIXME as a
logical decoding limitation and wishlist item; I'll amend to that effect.
+ pq_sendbyte(out, 'S'); /* message type field */ + pq_sendbyte(out, 1); /* startup message version */ For now protocol is 1, but for code readability it might be better to change this line to: + pq_sendbyte(out, PGLOGICAL_PROTO_VERSION_NUM); /* startup message version */
The startup message format isn't the same as the protocol version.
Hopefully we'll never have to change it. The reason it's specified is so
that if we ever do bump it a decoding plugin can recognise an old client
and fall back. Maybe it's BC overkill but I'd kick myself for not doing it
if we ever decided to (say) add support for structured json startup options
from the client. Working on BDR has taught me that there's no such thing as
too much consideration for cross-version compat and negotiation in
replication.
I'm happy to create a new define for that an comment to this effect.
Just for the sake of avoiding code repetition: + for (i = 0; i < desc->natts; i++) + { + if (desc->attrs[i]->attisdropped) + continue; + nliveatts++; + } + pq_sendint(out, nliveatts, 2);
The exact same code is in write_tuple and write_attrs. I don't know what's
policy for refactoring, but this might be extracted into separate function.
Seems trivial enough not to care, but probably can.
+ else if (att->attlen == -1) + { + char *data = DatumGetPointer(values[i]); + + /* send indirect datums inline */ + if (VARATT_IS_EXTERNAL_INDIRECT(values[i])) + { + struct varatt_indirect redirect; + VARATT_EXTERNAL_GET_POINTER(redirect, data); + data = (char *) redirect.pointer; + }
I really don't like this. We have function parameter "data" and now
are creating new variable with the same name.
I agree. Good catch.
I don't much like the use of 'data' as the param name for the plugin
private data and am quite inclined to change that instead, to
plugin_private or something.
pglogical_proto_json.c
+ appendStringInfo(out, ", \"origin_lsn\":\"%X/%X\"", + (uint32)(txn->origin_lsn >> 32), (uint32)(txn->origin_lsn)); I remember there was discussion on *-hackers recently about %X/%X; I'll try to find it and check whether it's according to final conclusion.
Thanks.
pglogical_relmetacache.c
First. In pglogical_output.c, in pg_decode_startup we are calling
init_relmetacache. I haven't found call to destroy_relmetacache
and comment says that it must be called at backend shutdown.
Is it guaranteed? Or will cache get freed with its context?
Hm, ok. It must never be called before shutdown, but it's not necessary to
call at all. I'll amend it. It's present just in case future Valgrind
support wants to call it to explicitly clean up memory.
I'll take a closer look there. I think I changed the lifetime of the
relmetacache after figuring out a better way to handle the cache
invalidations and it's possible I missed an update in the comments.
+ /* Find cached function info, creating if not found */ + hentry = (struct PGLRelMetaCacheEntry*) hash_search(RelMetaCache, + (void *)(&RelationGetRelid(rel)), + HASH_ENTER, &found); + + if (!found) + { + Assert(hentry->relid = RelationGetRelid(rel)); + hentry->is_cached = false; + hentry->api_private = NULL; + } + + Assert(hentry != NULL);
Shouldn't Assert be just after calling hash_search? We're (if !found)
dereferencing hentry and only after checking whether it's not NULL.
Yeah. It's more of an exit condition, stating "this function never returns
non-null hentry" but moving it up is fine. Can do.
I haven't found relevance of relmeta_cache_size attribute.
It's set to value coming from client - but then the only thing that matters
is whether it is 0 or not. I'll try to reread DESIGN.md session about
cache,
but for now (quite late and I'm quite tired) it is not clear on the first
reading.
It's the first part of a feature. You can turn the relation metadata cache
off, set it to unlimited, or (in future, not implemented yet) set a bounded
size where a LRU is used to evict the oldest entry.
The LRU approach is more complex since we have to track the LRU and do
evictions as well as the cache map its self. Efficiently. I expect this to
be a 1.1 or later feature. Having the param as an int now means we don't
later land up having to have two params, one to enable the cache and
another to bound its size, so once it's fully implemented it'll (IMO) be
clearer what's going on.
Thanks again for the detailed code examination. I find it hard to read
familiar code closely for review since I see what I expect to see. So it's
really, really useful.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
On Wed, Jan 20, 2016 at 8:04 PM, Craig Ringer <craig@2ndquadrant.com> wrote:
itself is an abbreviation of its self.
I do not think this is true.
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On Wed, Jan 20, 2016 at 12:54 AM, Craig Ringer <craig@2ndquadrant.com> wrote:
The idea here is that we want downwards compatibility as far as possible and
maintainable but we can't really be upwards compatible for breaking protocol
revisions. So the output plugin's native protocol version is inherently the
max protocol version and we don't need a separate MAX.
That idea seems right to me.
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
The following review has been posted through the commitfest application:
make installcheck-world: not tested
Implements feature: not tested
Spec compliant: not tested
Documentation: not tested
Documentation - although I haven't yet went through protocol documentation:
README.md
+ data stream. The output stream is designed to be compact and fast to decode,
+ and the plugin supports upstream filtering of data so that only the required
+ information is sent.
plugin supports upstream filtering of data through hooks so that ...
+ subset of that database may be selected for replication, currently based on
+ table and on replication origin. Filtering by a WHERE clause can be supported
+ easily in future.
Is this filtering by table and replication origin implemented? I haven't
noticed it in source.
+ other daemon is required. It's accumulated using
Stream of changes is accumulated...
+ [the `CREATE_REPLICATION_SLOT ... LOGICAL ...` or `START_REPLICATION SLOT ... LOGICAL ...` commands](http://www.postgresql.org/docs/current/static/logicaldecoding-walsender.html) to start streaming changes. (It can also be used via
+ [SQL level functions](http://www.postgresql.org/docs/current/static/logicaldecoding-sql.html)
+ over a non-replication connection, but this is mainly for debugging purposes)
Replication slot can also be configured (causing output plugin to be loaded) via [SQL level functions]...
+ The overall flow of client/server interaction is:
The overall flow of client/server interaction is as follows:
+ * Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'` if it's setting up for the first time
* Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'` to setup replication if it's connecting for the first time
+ Details are in the replication protocol docs.
Add link to file with protocol documentation.
+ If your application creates its own slots on first use and hasn't previously
+ connected to this database on this system you'll need to create a replication
+ slot. This keeps track of the client's replay state even while it's disconnected.
If your application hasn't previously connected to this database on this system
it'll need to create and configure replication slot which keeps track of the
client's replay state even while it's disconnected.
+ `pglogical`'s output plugin now sends a continuous series of `CopyData`
As this is separate plugin, use 'pglogical_output' plugin now sends...
(not only here but also in few other places).
+ All hooks are called in their own memory context, which lasts for the duration
All hooks are called in separate hook memory context, which lasts for the duration...
+ + switched to a longer lived memory context like TopMemoryContext. Memory allocated
+ + in the hook context will be automatically when the decoding session shuts down.
...will be automatically freed when the decoding...
DDL for global object changes must be synchronized via some external means.
Just:
Global object changes must be synchronized via some external means.
+ determine why an error occurs in a downstream, since you can examine a
+ json-ified representation of the xact. It's necessary to supply a minimal
since you can examine a transaction in json (and not binary) format. It's necessary
+ discard up to, as identifed by LSN (log sequence number). See
identified
+ Once you've peeked the stream and know the LSN you want to discard up to, you
+ can use `pg_logical_slot_peek_changes`, specifying an `upto_lsn`, to consume
Shouldn't it be pg_logical_slot_get_changes? get_changes consumes changes,
peek_changes leaves them in the stream. Especially as example below
points that we need to use get_changes.
+ tp to but not including that point, i.e. that will be the
+ point at which replay resumes.
IMO it's better to introduce new sentence:
that point. This will be the point at which replay resumes.
DESIGN.md:
+ attnos don't necessarily correspond. The column names might, and their ordering
+ might even be the same, but any column drop or column type change will result
The column names and their ordering might even be the same...
README.pglogical_output_plhooks:
+ Note that pglogical
+ must already be installed so that its headers can be found.
Note that pglogical_output must already...
+ Arguments are the oid of the affected relation, and the change type: 'I'nsert,
+ 'U'pdate or 'D'elete. There is no way to access the change data - columns changed,
+ new values, etc.
Is it true (no way to access change data)? You added passing change
to C hooks; from looking at code it looks like it's true, but I want to be sure.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 22 January 2016 at 06:13, Tomasz Rybak <tomasz.rybak@post.pl> wrote:
+ data stream. The output stream is designed to be compact and fast to decode, + and the plugin supports upstream filtering of data so that only the required + information is sent.plugin supports upstream filtering of data through hooks so that ...
Ok.
+ subset of that database may be selected for replication, currently based on + table and on replication origin. Filtering by a WHERE clause can be supported + easily in future.Is this filtering by table and replication origin implemented? I haven't
noticed it in source.
That's what the hooks are for.
+ other daemon is required. It's accumulated using
Stream of changes is accumulated...
Ok.
+ [the `CREATE_REPLICATION_SLOT ... LOGICAL ...` or `START_REPLICATION SLOT ... LOGICAL ...` commands]( http://www.postgresql.org/docs/current/static/logicaldecoding-walsender.html) to start streaming changes. (It can also be used via + [SQL level functions]( http://www.postgresql.org/docs/current/static/logicaldecoding-sql.html) + over a non-replication connection, but this is mainly for debugging purposes)Replication slot can also be configured (causing output plugin to be
loaded) via [SQL level functions]...
Covered in the next section. Or at least it is in the SGML docs conversion
I'm still trying to finish off..
+ * Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'`
if it's setting up for the first time* Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL 'pglogical'` to
setup replication if it's connecting for the first time
I disagree. It's entirely possible to do your slot creation/setup manually
or via something else, re-use a slot first created by another node, etc.
Slot creation is part of client setup, not so much connection.
+ Details are in the replication protocol docs.
Add link to file with protocol documentation.
Is done in SGML conversion.
+ If your application creates its own slots on first use and hasn't previously + connected to this database on this system you'll need to create a replication + slot. This keeps track of the client's replay state even while it's disconnected.If your application hasn't previously connected to this database on this
system
it'll need to create and configure replication slot which keeps track of
the
client's replay state even while it's disconnected.
As above, I don't quite agree.
+ `pglogical`'s output plugin now sends a continuous series of `CopyData`
As this is separate plugin, use 'pglogical_output' plugin now sends...
(not only here but also in few other places).
Thanks. Done.
+ All hooks are called in their own memory context, which lasts for the
duration
All hooks are called in separate hook memory context, which lasts for the
duration...
I don't see the difference, but OK.
Simplified:
All hooks are called in a memory context that lasts ...
+ + switched to a longer lived memory context like TopMemoryContext. Memory allocated + + in the hook context will be automatically when the decoding session shuts down....will be automatically freed when the decoding...
Fixed, thanks.
DDL for global object changes must be synchronized via some external means.
Just:
Global object changes must be synchronized via some external means.
Agree, done.
+ determine why an error occurs in a downstream, since you can examine a + json-ified representation of the xact. It's necessary to supply a minimalsince you can examine a transaction in json (and not binary) format. It's
necessary
Ok, done.
+ Once you've peeked the stream and know the LSN you want to discard up to, you + can use `pg_logical_slot_peek_changes`, specifying an `upto_lsn`, to consumeShouldn't it be pg_logical_slot_get_changes? get_changes consumes changes,
peek_changes leaves them in the stream. Especially as example below
points that we need to use get_changes.
Yes. It should. Whoops. Thanks.
DESIGN.md:
+ attnos don't necessarily correspond. The column names might, and their ordering + might even be the same, but any column drop or column type change will resultThe column names and their ordering might even be the same...
I disagree, that has a different meaning. It's also not really user-facing
docs so I'm not too worried about being quite as readable.
README.pglogical_output_plhooks:
+ Note that pglogical + must already be installed so that its headers can be found.Note that pglogical_output must already...
Thanks.
+ Arguments are the oid of the affected relation, and the change type: 'I'nsert, + 'U'pdate or 'D'elete. There is no way to access the change data - columns changed, + new values, etc.Is it true (no way to access change data)? You added passing change
to C hooks; from looking at code it looks like it's true, but I want to be
sure.
While the change data is now passed to the C hook, there's no attempt to
expose it via PL/PgSQL. So yeah, that's still true.
Thanks again for this. Sorry it's taken so long to get the SGML docs
converted. Too many other things on.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
I'm merging all your emails for sake of easier discussion.
I also cut all fragments that do not require response.
W dniu 22.01.2016, pią o godzinie 11∶06 +0800, użytkownik Craig Ringer
napisał:
We might also think about changing name of plugin to something
resembling "logical_streaming_decoder" or even "logical_streamer"
I'm open to ideas there but I'd want some degree of consensus before
undertaking the changes required.
I know that it'd require much changes (both in this and pglogical
plugin), and thus don't want to press for name change.
On one hand - changing of name might be good to avoid tight mental
coupling between pglogical_output and pglogica. At the same time - it's
much work, and I cannot think of any short and nice name, so
pglogical_output might stay IMO.
+ subset of that database may be selected for replication, currently based on + table and on replication origin. Filtering by a WHERE clause can be supported + easily in future.Is this filtering by table and replication origin implemented? I
haven't
noticed it in source.That's what the hooks are for.
Current documentation suggests that replicating only selected is
already available:
+ A subset of that database may be selected for replication, currently
+ based on table and on replication origin.
"currently based on table and on replication origin" means to me that
current state of plugin allows for just chosing which tables
to replicate. I'd see something like:
"A subset of that database might be selected for replication, e.g.
only chosen tables or changes from particular origin, in custom hook"
to convey that user needs to provide hook for filtering.
+ [the `CREATE_REPLICATION_SLOT ... LOGICAL ...` or `START_REPLICATION SLOT ... LOGICAL ...` commands](http://www.postg resql.org/docs/current/static/logicaldecoding-walsender.html) to start streaming changes. (It can also be used via + [SQL level functions](http://www.postgresql.org/docs/current/stat ic/logicaldecoding-sql.html) + over a non-replication connection, but this is mainly for debugging purposes)Replication slot can also be configured (causing output plugin to
be loaded) via [SQL level functions]...Covered in the next section. Or at least it is in the SGML docs
conversion I'm still trying to finish off..
OK, then I'll wait for the final version to review that.
+ * Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL
'pglogical'` if it's setting up for the first time* Client issues `CREATE_REPLICATION_SLOT slotname LOGICAL
'pglogical'` to setup replication if it's connecting for the first
timeI disagree. It's entirely possible to do your slot creation/setup
manually or via something else, re-use a slot first created by
another node, etc. Slot creation is part of client setup, not so much
connection.
I'd propose then:
' * Client issues "CREATE_REPLICATION_SLOT ..." if the replication
was not configured earler, e.g. during previous connection, or manually
via [SQL functions | link to documentation]"
+ If your application creates its own slots on first use and hasn't previously + connected to this database on this system you'll need to create a replication + slot. This keeps track of the client's replay state even while it's disconnected.If your application hasn't previously connected to this database on
this system
it'll need to create and configure replication slot which keeps
track of the
client's replay state even while it's disconnected.As above, I don't quite agree.
"If your application hasn't previously connedted to this database on
this system, and the replication slot was not configured through other
means (e.g. manually using [SQL functions | URL ] then you'll need
to create and configure replication slot ..."
DESIGN.md:+ attnos don't necessarily correspond. The column names might, and their ordering + might even be the same, but any column drop or column type change will resultThe column names and their ordering might even be the same...
I disagree, that has a different meaning. It's also not really user-
facing docs so I'm not too worried about being quite as readable.
I do not try to change meaning but fix grammar. I'd like to have here
either more or less commas. So either:
+ The column names (and their ordering) might ...
or:
+ The column names, and their ordering, might ...
Is it true (no way to access change data)? You added passing change
to C hooks; from looking at code it looks like it's true, but I
want to be sure.While the change data is now passed to the C hook, there's no attempt
to expose it via PL/PgSQL. So yeah, that's still true.
Thanks for confirming.
Speaking about flags - in most cases they are 0; only for
attributes
we might have:
+ flags |= IS_REPLICA_IDENTITY;
I assume that flags field is put into protocol for future needs?Correct. The protocol specifies (in most places; need to double check
all sites) that the lower 4 bits are reserved and must be treated as
an ERROR if set. The high 4 bits must be ignored if set and not
recognised. That gives us some room to wiggle without bumping the
protocol version incompatibly, and lets us use capabilities (via
client-supplied parameters) to add extra information on the wire.
Thanks for explanation. You're right - protocol.txt explains that
quite nicely.
+ /* FIXME support whole tuple (O tuple type) */
+ if (oldtuple != NULL) + { + pq_sendbyte(out, 'K'); /* old key follows */ + pglogical_write_tuple(out, data, rel, oldtuple); + } I don't fully understand. We are sending whole old tuple here, so this FIXME should be more about supporting sending just keys. But then comment "old key follows" is not true. Or am I missing something here?In wal_level=logical the tuple that's written is an abbreviated tuple
containing data only for the REPLICA IDENTITY fields.
I didn't know that - thanks for explanation.
+ pq_sendbyte(out, 'S'); /* message type field */ + pq_sendbyte(out, 1); /* startup message version */ For now protocol is 1, but for code readability it might be better to change this line to: + pq_sendbyte(out, PGLOGICAL_PROTO_VERSION_NUM); /* startup message version */The startup message format isn't the same as the protocol version.
Hopefully we'll never have to change it. The reason it's specified is
so that if we ever do bump it a decoding plugin can recognise an old
client and fall back. Maybe it's BC overkill but I'd kick myself for
not doing it if we ever decided to (say) add support for structured
json startup options from the client. Working on BDR has taught me
that there's no such thing as too much consideration for cross-
version compat and negotiation in replication.
Then I'd suggest adding named constant for this - so startup message
version is not mistaken for protocol version. Something like:
+ #define PGLOGICA_STARTUP_MESSAGE_VERSION_NUM 1
This way it is obvious that "1" and "1" do not mean that same.
Just for the sake of avoiding code repetition:
+ for (i = 0; i < desc->natts; i++) + { + if (desc->attrs[i]->attisdropped) + continue; + nliveatts++; + } + pq_sendint(out, nliveatts, 2); The exact same code is in write_tuple and write_attrs. I don't know what's policy for refactoring, but this might be extracted into separate function.Seems trivial enough not to care, but probably can.
IMO such code (computing number of live attributes in relation) should
be used often enough to deserve own function - isn't such function
available in PostgreSQL?
I really don't like this. We have function parameter "data" and
now
are creating new variable with the same name.I agree. Good catch.
I don't much like the use of 'data' as the param name for the plugin
private data and am quite inclined to change that instead, to
plugin_private or something.
Maybe rename function parameter to plugin_data or pluging_private_data?
+ /* Find cached function info, creating if not found */
+ hentry = (struct PGLRelMetaCacheEntry*) hash_search(RelMetaCache, + (void *)(&RelationGetRelid(rel)), + HASH_ENTER, &found); + + if (!found) + { + Assert(hentry->relid = RelationGetRelid(rel)); + hentry->is_cached = false; + hentry->api_private = NULL; + } + + Assert(hentry != NULL); Shouldn't Assert be just after calling hash_search? We're (if !found) dereferencing hentry and only after checking whether it's not NULL.Yeah. It's more of an exit condition, stating "this function never
returns non-null hentry" but moving it up is fine. Can do.
Please do so. It's more about conveying intent and increaing code
readability - so anyone knows that hentry should not be NULL
when returned by hash_search, and not some time later.
Thanks again for this. Sorry it's taken so long to get the SGML docs
converted. Too many other things on.
No problem, I know how it is.
Best regards.
--
Tomasz Rybak GPG/PGP key ID: 2AD5 9860
Fingerprint A481 824E 7DD3 9C0E C40A 488E C654 FB33 2AD5 9860
http://member.acm.org/~tomaszrybak
The following review has been posted through the commitfest application:
make installcheck-world: not tested
Implements feature: not tested
Spec compliant: not tested
Documentation: not tested
Final part of review:
protocol.txt
+|origin_identifier|signed char[origin_identifier_length]|An origin identifier of arbitrary, upstream-application-defined structure. _Should_ be text in the same encoding as the upstream database. NULL-terminated. _Should_ be 7-bit ASCII.
Does it need NULL-termination when previous field contains length of origin_identifier?
Similarly for relation metadata message.
+ metadata message. All consecutive row messages must currently have the same
+ relidentifier. (_Later extensions to add metadata caching will relax these
+ requirements for clients that advertise caching support; see the documentation
+ on metadata messages for more detail_).
Shouldn't this be changed as metadata cache is implemented?
+ |relidentifier|uint32|relidentifier that matches the table metadata message sent for this row.
+ (_Not present in BDR, which sends nspname and relname instead_)
and
+ |natts|uint16|Number of fields sent in this tuple part.
+ (_Present in BDR, but meaning significantly different here)_
Is BDR mention relevant here? It was not mentioned anywhere else, and now appears
ex machina.
Long quote - but required.
+ ==== Tuple fields
+
+ |===
+ |Tuple type|signed char|Identifies the kind of tuple being sent.
+
+ |tupleformat|signed char|‘**T**’ (0x54)
+ |natts|uint16|Number of fields sent in this tuple part.
+ (_Present in BDR, but meaning significantly different here)_
+ |[tuple field values]|[composite]|
+ |===
+
+ ===== Tuple tupleformat compatibility
+
+ Unrecognised _tupleformat_ kinds are a protocol error for the downstream.
+
+ ==== Tuple field value fields
+
+ These message parts describe individual fields within a tuple.
+
+ There are two kinds of tuple value fields, abbreviated and full. Which is being
+ read is determined based on the first field, _kind_.
+
+ Abbreviated tuple value fields are nothing but the message kind:
+
+ |===
+ |*Message*|*Type/Size*|*Notes*
+
+ |kind|signed char| * ‘**n**’ull (0x6e) field
+ |===
+
+ Full tuple value fields have a length and datum:
+
+ |===
+ |*Message*|*Type/Size*|*Notes*
+
+ |kind|signed char| * ‘**i**’nternal binary (0x62) field
+ |length|int4|Only defined for kind = i\|b\|t
+ |data|[length]|Data in a format defined by the table metadata and column _kind_.
+ |===
+
+ ===== Tuple field values kind compatibility
+
+ Unrecognised field _kind_ values are a protocol error for the downstream. The
+ downstream may not continue processing the protocol stream after this
+ point**.**
+
+ The upstream may not send ‘**i**’nternal or ‘**b**’inary format values to the
+ downstream without the downstream negotiating acceptance of such values. The
+ downstream will also generally negotiate to receive type information to use to
+ decode the values. See the section on startup parameters and the startup
+ message for details.
I do not fully get it.
For each tuple we are supposed to have "Tuple type" (which is kind?). Does it
mean that T1 might be sent using "i" kind and T2 sent using "b" kind?
At the same tme we have kind "n" (null) - but it belongs to field level
(one field might be null, not entire tuple).
In other words - do we have "i" and then "T" and then number of attributes,
or "T', then number of attributes, then "i" or "b" or "n" for each of attributes?
Also - description of "b" seems missing.
+ Before sending changed rows for a relation, a metadata message for the relation
+ must be sent so the downstream knows the namespace, table name, column names,
+ optional column types, etc. A relidentifier field, an arbitrary numeric value
+ unique for that relation on that upstream connection, maps the metadata to
+ following rows.
+
+ A client should not assume that relation metadata will be followed immediately
+ (or at all) by rows, since future changes may lead to metadata messages being
+ delivered at other times. Metadata messages may arrive during or between
+ transactions.
+
+ The upstream may not assume that the downstream retains more metadata than the
+ one most recent table metadata message. This applies across all tables, so a
+ client is permitted to discard metadata for table x when getting metadata for
+ table y. The upstream must send a new metadata message before sending rows for
+ a different table, even if that metadata was already sent in the same session
+ or even same transaction. _This requirement will later be weakened by the
+ addition of client metadata caching, which will be advertised to the upstream
+ with an output plugin parameter._
This needs reworking while metadata caching is supported
+ |Message type|signed char|‘**S**’ (0x53) - startup
+ |Startup message version|uint8|Value is always “1”.
Value is "1" for the current plugin version. It is represented in code as
PGLOGICAL_STARTUP_MESSAGE_VERSION_NUM.
+ |startup_params_format|int8|1|The format version of this startup parameter set. Always the digit 1 (0x31), null terminated.
int8 suggests binary value, and here we are sending ASCII which is NULL-terminated.
It's a bit inconsistent.
Please wrap long lines in the last part of protocol.txt file, starting with
"Arguments client supplies to output plugn" section.
Also - please put 3 paragraphs from your email from 2016-01-07 15:50
(staring with "but this isn't just about replication") into README.
This is really good rationale for this plugin existence.
Best regards.
The new status of this patch is: Waiting on Author
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 2016-01-18 21:47:27 +0000, Tomasz Rybak wrote:
We might also think about changing name of plugin to something resembling "logical_streaming_decoder" or even "logical_streamer"
FWIW, I find those proposals unconvincing. Not that pglogical_output is
grand, but "streaming decoder" or "logical_streamer" aren't even
correct. And output plugin isn't a "decoder" or a "streamer".
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Hi,
so, I'm reviewing the output of:
git diff $(git merge-base upstream/master 2ndq/dev/pglogical-output)..2ndq/dev/pglogical-output diff --git a/contrib/Makefile b/contrib/Makefile index bd251f6..028fd9a 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -35,6 +35,8 @@ SUBDIRS = \ pg_stat_statements \ pg_trgm \ pgcrypto \ + pglogical_output \ + pglogical_output_plhooks \
I'm doubtful we want these plhooks. You aren't allowed to access normal
(non user catalog) tables in output plugins. That seems too much to
expose to plpgsql function imo.
+++ b/contrib/pglogical_output/README.md
I don't think we've markdown in postgres so far - so let's just keep the
current content and remove the .md :P
+==== Table metadata header + +|=== +|*Message*|*Type/Size*|*Notes* + +|Message type|signed char|Literal ‘**R**’ (0x52) +|flags|uint8| * 0-6: Reserved, client _must_ ERROR if set and not recognised. +|relidentifier|uint32|Arbitrary relation id, unique for this upstream. In practice this will probably be the upstream table’s oid, but the downstream can’t assume anything. +|nspnamelength|uint8|Length of namespace name +|nspname|signed char[nspnamelength]|Relation namespace (null terminated) +|relnamelength|uint8|Length of relation name +|relname|char[relname]|Relation name (null terminated) +|attrs block|signed char|Literal: ‘**A**’ (0x41) +|natts|uint16|number of attributes +|[fields]|[composite]|Sequence of ‘natts’ column metadata blocks, each of which begins with a column delimiter followed by zero or more column metadata blocks, each with the same column metadata block header.
That's a fairly high overhead. Hm.
+== JSON protocol + +If `proto_format` is set to `json` then the output plugin will emit JSON +instead of the custom binary protocol. JSON support is intended mainly for +debugging and diagnostics. +
I'm fairly strongly opposed to including two formats in one output
plugin. I think the demand for being able to look into the binary
protocol should instead be satisfied by having a function that "expands"
the binary data returned into something easier to understand.
+ * Copyright (c) 2012-2015, PostgreSQL Global Development Group
2016 ;)
+ case PARAM_BINARY_BASETYPES_MAJOR_VERSION: + val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_UINT32); + data->client_binary_basetypes_major_version = DatumGetUInt32(val); + break;
Why is the major version tied to basetypes (by name)? Seem more
generally useful.
+ case PARAM_RELMETA_CACHE_SIZE: + val = get_param_value(elem, false, OUTPUT_PARAM_TYPE_INT32); + data->client_relmeta_cache_size = DatumGetInt32(val); + break;
I'm not convinced this a) should be optional b) should have a size
limit. Please argue for that choice. And how the client should e.g. know
about evictions in that cache.
--- /dev/null +++ b/contrib/pglogical_output/pglogical_config.h @@ -0,0 +1,55 @@ +#ifndef PG_LOGICAL_CONFIG_H +#define PG_LOGICAL_CONFIG_H + +#ifndef PG_VERSION_NUM +#error <postgres.h> must be included first +#endif
Huh?
+#include "nodes/pg_list.h" +#include "pglogical_output.h" + +inline static bool +server_float4_byval(void) +{ +#ifdef USE_FLOAT4_BYVAL + return true; +#else + return false; +#endif +} + +inline static bool +server_float8_byval(void) +{ +#ifdef USE_FLOAT8_BYVAL + return true; +#else + return false; +#endif +} + +inline static bool +server_integer_datetimes(void) +{ +#ifdef USE_INTEGER_DATETIMES + return true; +#else + return false; +#endif +} + +inline static bool +server_bigendian(void) +{ +#ifdef WORDS_BIGENDIAN + return true; +#else + return false; +#endif +}
Not convinced these should exists, and even moreso exposed in a header.
+/* + * Returns Oid of the hooks function specified in funcname. + * + * Error is thrown if function doesn't exist or doen't return correct datatype + * or is volatile. + */ +static Oid +get_hooks_function_oid(List *funcname) +{ + Oid funcid; + Oid funcargtypes[1]; + + funcargtypes[0] = INTERNALOID; + + /* find the the function */ + funcid = LookupFuncName(funcname, 1, funcargtypes, false); + + /* Validate that the function returns void */ + if (get_func_rettype(funcid) != VOIDOID) + { + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("function %s must return void", + NameListToString(funcname)))); + }
Hm, this seems easy to poke holes into. I mean you later use it like:
+ if (data->hooks_setup_funcname != NIL) + { + hooks_func = get_hooks_function_oid(data->hooks_setup_funcname); + + old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt); + (void) OidFunctionCall1(hooks_func, PointerGetDatum(&data->hooks)); + MemoryContextSwitchTo(old_ctxt);
e.g. you basically assume the function the does something reasonable
with those types. Why don't we instead create a 'plogical_hooks' return
type, and have the function return that?
+ if (func_volatile(funcid) == PROVOLATILE_VOLATILE) + { + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("function %s must not be VOLATILE", + NameListToString(funcname)))); + }
Hm, not sure what that's supposed to achieve. You could argue for
requiring the function to be immutable (i.e. not stable or volatile),
but I'm not sure what that'd achieve.
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt); + (void) (*data->hooks.startup_hook)(&args); + MemoryContextSwitchTo(old_ctxt);
What is the hooks memory contexts intended to achieve? It's apparently
never reset. Normally output plugin calbacks are called in more
shortlived memory contexts, for good reason, to avoid leaks....
+bool +call_row_filter_hook(PGLogicalOutputData *data, ReorderBufferTXN *txn, + Relation rel, ReorderBufferChange *change) +{ + struct PGLogicalRowFilterArgs hook_args; + MemoryContext old_ctxt; + bool ret = true; + + if (data->hooks.row_filter_hook != NULL) + { + hook_args.change_type = change->action; + hook_args.private_data = data->hooks.hooks_private_data; + hook_args.changed_rel = rel; + hook_args.change = change; + + elog(DEBUG3, "calling pglogical row filter hook"); + + old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt); + ret = (*data->hooks.row_filter_hook)(&hook_args);
Why aren't we passing txn to the filter? ISTM it'd be better to
basically reuse/extend the signature by the the original change
callback.
+/* These must be available to pg_dlsym() */
No the following don't? And they aren't, since they're static functions?
_PG_init and _PG_output_plugin_init need to, but that's it.
+/* + * COMMIT callback + */ +void +pg_decode_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, + XLogRecPtr commit_lsn) +{
Missing static's?
+/* + * Relation metadata invalidation, for when a relcache invalidation + * means that we need to resend table metadata to the client. + */ +static void +relmeta_cache_callback(Datum arg, Oid relid) + { + /* + * We can be called after decoding session teardown becaues the + * relcache callback isn't cleared. In that case there's no action + * to take. + */ + if (RelMetaCache == NULL) + return; + + /* + * Nobody keeps pointers to entries in this hash table around so + * it's safe to directly HASH_REMOVE the entries as soon as they are + * invalidated. Finding them and flagging them invalid then removing + * them lazily might save some memory churn for tables that get + * repeatedly invalidated and re-sent, but it dodesn't seem worth + * doing. + * + * Getting invalidations for relations that aren't in the table is + * entirely normal, since there's no way to unregister for an + * invalidation event. So we don't care if it's found or not. + */ + (void) hash_search(RelMetaCache, &relid, HASH_REMOVE, NULL); + }
So, I don't buy this, like at all. The cache entry is passed to
functions, while we call output functions and such. Which in turn can
cause cache invalidations to be processed.
+struct PGLRelMetaCacheEntry +{ + Oid relid; + /* Does the client have this relation cached? */ + bool is_cached; + /* Field for API plugin use, must be alloc'd in decoding context */ + void *api_private; +};
I don't see how api_private can safely be used. At the very least it
needs a lot more documentation about memory lifetime rules and
such. Afaics we'd just forever leak memory atm.
Greetings,
Andres Freund
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 29 January 2016 at 18:16, Andres Freund <andres@anarazel.de> wrote:
Hi,
so, I'm reviewing the output of:
Thankyou very much for the review.
+ pglogical_output_plhooks \
I'm doubtful we want these plhooks. You aren't allowed to access normal
(non user catalog) tables in output plugins. That seems too much to
expose to plpgsql function imo.
You're right. We've got no way to make sure the user sticks to things
that're reasonably safe.
The intent of that module was to allow people to write row and origin
filters in plpgsql, to serve as an example of how to implement hooks, and
to provide a tool usable in testing pglogical_output from pg_regress.
An example can be in C, since it's not safe to do it in plpgsql as you
noted. A few toy functions will be sufficient for test use.
As for allowing users to flexibly filter, I'm stating to think that hooks
in pglogical_output aren't really the best long term option. They're needed
for now, but for 9.7+ we should look at whether it's practical to separate
"what gets forwarded" policy from the mechanics of how it gets sent.
pglogical_output currently just farms part of the logical decoding hook out
to its own hooks, but it wouldn't have to do that if logical decoding let
you plug in policy on what you send separately to how you send it. Either
via catalogs or plugin functions separate to the output plugin.
(Kinda off topic, though, and I think we need the hooks for now, just not
the plpgsql implementation).
+++ b/contrib/pglogical_output/README.mdI don't think we've markdown in postgres so far - so let's just keep the
current content and remove the .md
I'm halfway through turning it all into SGML anyway. I just got sidetracked
by other work. I'd be just as happy to leave it as markdown but figured
SGML would likely be required.
+==== Table metadata header + +|=== +|*Message*|*Type/Size*|*Notes* + +|Message type|signed char|Literal ‘**R**’ (0x52) +|flags|uint8| * 0-6: Reserved, client _must_ ERROR if set and notrecognised.
+|relidentifier|uint32|Arbitrary relation id, unique for this upstream.
In practice this will probably be the upstream table’s oid, but the
downstream can’t assume anything.+|nspnamelength|uint8|Length of namespace name +|nspname|signed char[nspnamelength]|Relation namespace (null terminated) +|relnamelength|uint8|Length of relation name +|relname|char[relname]|Relation name (null terminated) +|attrs block|signed char|Literal: ‘**A**’ (0x41) +|natts|uint16|number of attributes +|[fields]|[composite]|Sequence of ‘natts’ column metadata blocks, eachof which begins with a column delimiter followed by zero or more column
metadata blocks, each with the same column metadata block header.That's a fairly high overhead. Hm.
Yeah, and it shows in Oleksandr's measurements. However, that's a metadata
message that is sent only pretty infrequently if you enable relation
metadata caching. Which is really necessary to get reasonable performance
on anything but the simplest workloads, and is only optional because it
makes it easier to write and test a client without it first.
+== JSON protocol + +If `proto_format` is set to `json` then the output plugin will emit JSON +instead of the custom binary protocol. JSON support is intended mainlyfor
+debugging and diagnostics.
+I'm fairly strongly opposed to including two formats in one output
plugin. I think the demand for being able to look into the binary
protocol should instead be satisfied by having a function that "expands"
the binary data returned into something easier to understand.
Per our discussion yesterday I think I agree with you on that now.
My thinking is that we should patch pg_recvlogical to be able to load a
decoder plugin. Then extract the protocol decoding support from pglogical
into a separately usable library that can be loaded by pg_recvlogical,
pglogical downstream, and by SQL-level debug/test helper functions.
pg_recvlogical won't be able to decode binary or internal format field
values, but you can simply not request that they be sent.
+ case PARAM_BINARY_BASETYPES_MAJOR_VERSION: + val = get_param_value(elem, false,OUTPUT_PARAM_TYPE_UINT32);
+
data->client_binary_basetypes_major_version = DatumGetUInt32(val);
+ break;
Why is the major version tied to basetypes (by name)? Seem more
generally useful.
I found naming that param rather awkward.
The idea is that we can rely on the Pg major version only for types defined
in core. It's mostly safe for built-in extensions in that few if any
people ever upgrade them, but it's not strictly correct even there. Most of
them (hstore, etc) don't expose their own versions so it's hard to know
what to do about them.
What I want(ed?) to do is let a downstream enumerate the extensions it has
and the version(s) it knows they're compatible with for send/recv and
internal formats. But designing that was going to be hairy in the time
available, and I think falling back on text representation for contribs and
extensions is the safest choice. A solid way to express an extension
compatibility matrix seemed rather too much to bite off as well as
everything else.
+ case PARAM_RELMETA_CACHE_SIZE: + val = get_param_value(elem, false,OUTPUT_PARAM_TYPE_INT32);
+ data->client_relmeta_cache_size =
DatumGetInt32(val);
+ break;
I'm not convinced this a) should be optional b) should have a size
limit. Please argue for that choice. And how the client should e.g. know
about evictions in that cache.
I don't think metadata every row makes sense. So it shouldn't be optional
in that sense. It was optional mostly because the downstream didn't
initially support it and I thought it'd be useful to allow people to write
simpler clients.
The cache logic is really quite simple though, so I'm happy enough to make
it required.
Re the size limit, my thinking was that there are (unfortunately) real
world apps that create and drop tables in production, that have tens of
thousands or more schemas that each have hundreds or more tables, etc.
They're awful and I wish people wouldn't do that, but they do. Rather than
expecting some unknown and unbounded cache size being able to limit the
number of tables for which the downstream is required to retain metadata
seemed possibly useful. I'm far from wedded to the idea as it requires the
upstream to maintain a metadata cache LRU (which it doesn't yet) and send
cache eviction messages to the downstream. Cutting that whole idea out
would simplify things.
--- /dev/null +++ b/contrib/pglogical_output/pglogical_config.h @@ -0,0 +1,55 @@ +#ifndef PG_LOGICAL_CONFIG_H +#define PG_LOGICAL_CONFIG_H + +#ifndef PG_VERSION_NUM +#error <postgres.h> must be included first +#endifHuh?
It's a stray that should've been part of pglogical/compat.h (which will
presumably get cut for inclusion in contrib anyway).
+#include "nodes/pg_list.h" +#include "pglogical_output.h" + +inline static bool +server_float4_byval(void) +{ +#ifdef USE_FLOAT4_BYVAL + return true; +#else + return false; +#endif +} + +inline static bool +server_float8_byval(void) +{ +#ifdef USE_FLOAT8_BYVAL + return true; +#else + return false; +#endif +} + +inline static bool +server_integer_datetimes(void) +{ +#ifdef USE_INTEGER_DATETIMES + return true; +#else + return false; +#endif +} + +inline static bool +server_bigendian(void) +{ +#ifdef WORDS_BIGENDIAN + return true; +#else + return false; +#endif +}Not convinced these should exists, and even moreso exposed in a header.
Yeah. The info's needed in both pglogical_config.c and pglogical_output.c
but that can probably be avoided.
+/* + * Returns Oid of the hooks function specified in funcname. + * + * Error is thrown if function doesn't exist or doen't return correctdatatype
+ * or is volatile. + */ +static Oid +get_hooks_function_oid(List *funcname) +{ + Oid funcid; + Oid funcargtypes[1]; + + funcargtypes[0] = INTERNALOID; + + /* find the the function */ + funcid = LookupFuncName(funcname, 1, funcargtypes, false); + + /* Validate that the function returns void */ + if (get_func_rettype(funcid) != VOIDOID) + { + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("function %s must return void", +NameListToString(funcname))));
+ }
Hm, this seems easy to poke holes into. I mean you later use it like:
+ if (data->hooks_setup_funcname != NIL) + { + hooks_func =get_hooks_function_oid(data->hooks_setup_funcname);
+ + old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt); + (void) OidFunctionCall1(hooks_func,PointerGetDatum(&data->hooks));
+ MemoryContextSwitchTo(old_ctxt);
e.g. you basically assume the function the does something reasonable
with those types. Why don't we instead create a 'plogical_hooks' return
type, and have the function return that?
I want to avoid requiring any extension to be loaded for the output plugin,
to just have it provide the mechanism for transporting data changes and not
start adding types, user catalogs, etc.
That's still OK if the signature in pg_proc is 'returns internal', I just
don't want anything visible from SQL.
My other reasons for this approach have been obsoleted now that we're
installing the pglogical/hooks.h header in Pg's server includes. Originally
I was afraid extensions would have to make a *copy* of the hooks struct
definition in their own headers. In that case we'd be able to add entries
only strictly at the end of the struct and could only use it by palloc0'ing
it and passing a pointer. Of course, I didn't think of the case where the
hook defining extension had the bigger of the two definitions...
So yeah. Happy to just return 'internal', DatumGetPointer it and cast to a
pointer to our hooks struct.
+ if (func_volatile(funcid) == PROVOLATILE_VOLATILE) + { + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("function %s must not be VOLATILE", +NameListToString(funcname))));
+ }
Hm, not sure what that's supposed to achieve. You could argue for
requiring the function to be immutable (i.e. not stable or volatile),
but I'm not sure what that'd achieve.
It's a stupid holdover from the function's use elsewhere, and entirely my
fault.
+ old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt); + (void) (*data->hooks.startup_hook)(&args); + MemoryContextSwitchTo(old_ctxt);What is the hooks memory contexts intended to achieve? It's apparently
never reset. Normally output plugin calbacks are called in more
shortlived memory contexts, for good reason, to avoid leaks....
Mainly to make memory allocated by and used by hooks more visible when
debugging, rather than showing it conflated with the main logical decoding
context, while still giving hooks somewhere to store state that they may
need across the decoding session.
Not going to argue strongly for it. Just seemed like something I'd regret
not having when trying to figure out why the decoding context was massive
for no apparent reason later...
+bool
+call_row_filter_hook(PGLogicalOutputData *data, ReorderBufferTXN *txn, + Relation rel, ReorderBufferChange *change) +{ + struct PGLogicalRowFilterArgs hook_args; + MemoryContext old_ctxt; + bool ret = true; + + if (data->hooks.row_filter_hook != NULL) + { + hook_args.change_type = change->action; + hook_args.private_data = data->hooks.hooks_private_data; + hook_args.changed_rel = rel; + hook_args.change = change; + + elog(DEBUG3, "calling pglogical row filter hook"); + + old_ctxt = MemoryContextSwitchTo(data->hooks_mctxt); + ret = (*data->hooks.row_filter_hook)(&hook_args);Why aren't we passing txn to the filter? ISTM it'd be better to
basically reuse/extend the signature by the the original change
callback.
Yeah, probably.
I somewhat wish the original callbacks used struct arguments. Otherwise you
land up with fun #ifdef's when supporting multiple Pg versions and it's
hard to add new params. Quite annoying when dealing with extensions. OTOH
it's presumably faster to use the usual C calling convention.
+/* These must be available to pg_dlsym() */
No the following don't? And they aren't, since they're static functions?
_PG_init and _PG_output_plugin_init need to, but that's it.
Yeah. Total thinko.
+/* + * COMMIT callback + */ +void +pg_decode_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, + XLogRecPtr commit_lsn) +{Missing static's?
Yes.
+ /* + * Nobody keeps pointers to entries in this hash table around so + * it's safe to directly HASH_REMOVE the entries as soon as theyare
+ * invalidated. Finding them and flagging them invalid then
removing
+ * them lazily might save some memory churn for tables that get + * repeatedly invalidated and re-sent, but it dodesn't seem worth + * doing. + * + * Getting invalidations for relations that aren't in the table is + * entirely normal, since there's no way to unregister for an + * invalidation event. So we don't care if it's found or not. + */ + (void) hash_search(RelMetaCache, &relid, HASH_REMOVE, NULL); + }So, I don't buy this, like at all. The cache entry is passed to
functions, while we call output functions and such. Which in turn can
cause cache invalidations to be processed.
That was a misunderstanding on my part. I hadn't realised that cache
invalidations could be fired so easily by apparently innocuous actions, and
had assumed incorrectly that we'd get invalidations only outside the scope
of a decoding callback, not during one.
Clearly need to fix this with the usual invalid flag based approach. Which
in turn makes me agree with your proposal yesterday about adding a generic
mechanism for extensions to register their interest in invalidations on
tables, attach data to them, and not worry about the details of managing
the hash table correctly etc.
+struct PGLRelMetaCacheEntry +{ + Oid relid; + /* Does the client have this relation cached? */ + bool is_cached; + /* Field for API plugin use, must be alloc'd in decoding context */ + void *api_private; +};I don't see how api_private can safely be used. At the very least it
needs a lot more documentation about memory lifetime rules and
such. Afaics we'd just forever leak memory atm.
Yeah. It's pretty much just wrong, and since I don't have a compelling
reason for it I'm happy enough for it to just go away. Doing it correctly
would pretty much require a callback to be registered for freeing it, and
... meh.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
On Thu, Jan 7, 2016 at 4:50 PM, Craig Ringer <craig@2ndquadrant.com> wrote:
On 7 January 2016 at 01:17, Peter Eisentraut <peter_e@gmx.net> wrote:
On 12/22/15 4:55 AM, Craig Ringer wrote:
and we could probably go through them
one by one and ask, why do we need this bit? So that kind of system
will be very hard to review as a standalone submission.Again, I disagree. I think you're looking at this way too narrowly.
I find it quite funny, actually. Here we go and produce something that's a
nice re-usable component that other people can use in their products and
solutions ... and all anyone does is complain that the other part required
to use it as a canned product isn't posted yet (though it is now). But with
BDR all anyone ever does is complain that it's too tightly coupled to the
needs of a single product and the features extracted from it, like
replication origins, should be more generic and general purpose so other
people can use them in their products too. Which is it going to be?
As far as I can see, this patch is leveraging the current
infrastructure in core, logical decoding to convert the data obtained
as a set JSON blobs via a custom protocol. Its maintenance load looks
minimal, that's at least a good thing.
It would be helpful if you could take a step back and describe what *you*
think logical replication for PostgreSQL should look like. You clearly have
a picture in mind of what it should be, what requirements it satisfies, etc.
If you're going to argue based on that it'd be very helpful to describe it.
I might've missed some important points you've seen and you might've
overlooked issues I've seen.
Personally, if I would put a limit of what should be in-core, or what
should be logical replication from the core prospective, that would be
just to give to potential consumers (understand plugins here) of this
binary data (be it pg_ddl_command or what the logical decoding context
offers) a way to access it and then to allow those plugins to change
this binary data into something that is suited to it, and have simple
tools and example to test those things without relying on anything
external. test_decoding and test_ddl_deparse cover that already. What
I find a bit disturbing regarding this patch is: why would a JSON
representation be able to cover all kinds of needs? Aren't other
replication solution going to have their own data format and their own
protocol with different requirements?
Considering that this module could have a happier life if managed independently.
--
Michael
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 2016-01-31 05:09:33 +0800, Craig Ringer wrote:
On 29 January 2016 at 18:16, Andres Freund <andres@anarazel.de> wrote:
Hi,
so, I'm reviewing the output of:
Thankyou very much for the review.
Afaics you've not posted an updated version of this? Any chance you
could?
Greetings,
Andres Freund
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
On 15 March 2016 at 04:48, Andres Freund <andres@anarazel.de> wrote:
On 2016-01-31 05:09:33 +0800, Craig Ringer wrote:
On 29 January 2016 at 18:16, Andres Freund <andres@anarazel.de> wrote:
Hi,
so, I'm reviewing the output of:
Thankyou very much for the review.
Afaics you've not posted an updated version of this? Any chance you
could?
I'll try to get to it soon, yeah. I have been focusing on things that
cannot exist as extensions, especially timeline following for logical slots
and failover slots.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
On 15 March 2016 at 04:48, Andres Freund <andres@anarazel.de> wrote:
On 2016-01-31 05:09:33 +0800, Craig Ringer wrote:
On 29 January 2016 at 18:16, Andres Freund <andres@anarazel.de> wrote:
Hi,
so, I'm reviewing the output of:
Thankyou very much for the review.
Afaics you've not posted an updated version of this? Any chance you
could?
Here's v5.
It still needs json support to be removed and the plpgsql hooks replaced,
but the rest should be sorted out.
--
Craig Ringer http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services
Attachments:
0001-Timeline-following-for-logical-decoding.patchtext/x-patch; charset=US-ASCII; name=0001-Timeline-following-for-logical-decoding.patchDownload
From e1fb318e4c25b93770c53425277f3efa83ccb618 Mon Sep 17 00:00:00 2001
From: Craig Ringer <craig@2ndquadrant.com>
Date: Wed, 23 Mar 2016 09:03:04 +0800
Subject: [PATCH] Timeline following for logical decoding
Allow logical slots to follow timeline switches
Make logical replication slots timeline-aware, so replay can
continue from a historical timeline onto the server's current
timeline.
This is required to make failover slots possible and may also
be used by extensions that CreateReplicationSlot on a standby
and replay from that slot once the replica is promoted.
This does NOT add support for replaying from a logical slot on
a standby or for syncing slots to replicas.
---
src/backend/access/transam/xlogreader.c | 51 +++-
src/backend/access/transam/xlogutils.c | 260 ++++++++++++++++--
src/backend/replication/logical/logicalfuncs.c | 33 ++-
src/include/access/xlogreader.h | 37 ++-
src/include/access/xlogutils.h | 2 +-
src/test/modules/Makefile | 1 +
src/test/modules/test_slot_timelines/.gitignore | 3 +
src/test/modules/test_slot_timelines/Makefile | 22 ++
src/test/modules/test_slot_timelines/README | 19 ++
.../expected/load_extension.out | 19 ++
.../test_slot_timelines/sql/load_extension.sql | 7 +
.../test_slot_timelines--1.0.sql | 16 ++
.../test_slot_timelines/test_slot_timelines.c | 133 +++++++++
.../test_slot_timelines/test_slot_timelines.conf | 2 +
.../test_slot_timelines.control | 5 +
src/test/recovery/Makefile | 2 +
.../recovery/t/006_logical_decoding_timelines.pl | 304 +++++++++++++++++++++
17 files changed, 865 insertions(+), 51 deletions(-)
create mode 100644 src/test/modules/test_slot_timelines/.gitignore
create mode 100644 src/test/modules/test_slot_timelines/Makefile
create mode 100644 src/test/modules/test_slot_timelines/README
create mode 100644 src/test/modules/test_slot_timelines/expected/load_extension.out
create mode 100644 src/test/modules/test_slot_timelines/sql/load_extension.sql
create mode 100644 src/test/modules/test_slot_timelines/test_slot_timelines--1.0.sql
create mode 100644 src/test/modules/test_slot_timelines/test_slot_timelines.c
create mode 100644 src/test/modules/test_slot_timelines/test_slot_timelines.conf
create mode 100644 src/test/modules/test_slot_timelines/test_slot_timelines.control
create mode 100644 src/test/recovery/t/006_logical_decoding_timelines.pl
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index fcb0872..b7d249c 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -10,6 +10,9 @@
*
* NOTES
* See xlogreader.h for more notes on this facility.
+ *
+ * This file is compiled as both front-end and backend code, so it
+ * may not use ereport, server-defined static variables, etc.
*-------------------------------------------------------------------------
*/
@@ -116,6 +119,11 @@ XLogReaderAllocate(XLogPageReadCB pagereadfunc, void *private_data)
return NULL;
}
+#ifndef FRONTEND
+ /* Will be loaded on first read */
+ state->timelineHistory = NIL;
+#endif
+
return state;
}
@@ -135,6 +143,10 @@ XLogReaderFree(XLogReaderState *state)
pfree(state->errormsg_buf);
if (state->readRecordBuf)
pfree(state->readRecordBuf);
+#ifndef FRONTEND
+ if (state->timelineHistory)
+ list_free_deep(state->timelineHistory);
+#endif
pfree(state->readBuf);
pfree(state);
}
@@ -192,6 +204,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
{
XLogRecord *record;
XLogRecPtr targetPagePtr;
+ /*
+ * When validating headers we need to check the pre-link only if we're
+ * reading sequentally; see ValidXLogRecordHeader.
+ */
bool randAccess = false;
uint32 len,
total_len;
@@ -208,6 +224,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
if (RecPtr == InvalidXLogRecPtr)
{
+ /* No explicit start point; read the record after the one we just read */
RecPtr = state->EndRecPtr;
if (state->ReadRecPtr == InvalidXLogRecPtr)
@@ -223,11 +240,13 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/*
+ * Caller supplied a position to start at.
+ *
* In this case, the passed-in record pointer should already be
* pointing to a valid record starting position.
*/
Assert(XRecOffIsValid(RecPtr));
- randAccess = true; /* allow readPageTLI to go backwards too */
+ randAccess = true;
}
state->currRecPtr = RecPtr;
@@ -309,8 +328,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* XXX: more validation should be done here */
if (total_len < SizeOfXLogRecord)
{
- report_invalid_record(state, "invalid record length at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ report_invalid_record(state,
+ "invalid record length at %X/%X: wanted %lu, got %u",
+ (uint32) (RecPtr >> 32), (uint32) RecPtr,
+ SizeOfXLogRecord, total_len);
goto err;
}
gotheader = false;
@@ -466,9 +487,7 @@ err:
* Invalidate the xlog page we've cached. We might read from a different
* source after failure.
*/
- state->readSegNo = 0;
- state->readOff = 0;
- state->readLen = 0;
+ XLogReaderInvalCache(state);
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
@@ -580,10 +599,19 @@ ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
return readLen;
err:
+ XLogReaderInvalCache(state);
+ return -1;
+}
+
+/*
+ * Invalidate the xlogreader's cached page to force a re-read.
+ */
+void
+XLogReaderInvalCache(XLogReaderState *state)
+{
state->readSegNo = 0;
state->readOff = 0;
state->readLen = 0;
- return -1;
}
/*
@@ -600,8 +628,9 @@ ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
if (record->xl_tot_len < SizeOfXLogRecord)
{
report_invalid_record(state,
- "invalid record length at %X/%X",
- (uint32) (RecPtr >> 32), (uint32) RecPtr);
+ "invalid record length at %X/%X: wanted %lu, got %u",
+ (uint32) (RecPtr >> 32), (uint32) RecPtr,
+ SizeOfXLogRecord, record->xl_tot_len);
return false;
}
if (record->xl_rmid > RM_MAX_ID)
@@ -907,11 +936,9 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
err:
out:
/* Reset state to what we had before finding the record */
- state->readSegNo = 0;
- state->readOff = 0;
- state->readLen = 0;
state->ReadRecPtr = saved_state.ReadRecPtr;
state->EndRecPtr = saved_state.EndRecPtr;
+ XLogReaderInvalCache(state);
return found;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 444e218..537a2f5 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -21,6 +21,7 @@
#include "miscadmin.h"
+#include "access/timeline.h"
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogutils.h"
@@ -638,8 +639,17 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
}
/*
- * TODO: This is duplicate code with pg_xlogdump, similar to walsender.c, but
- * we currently don't have the infrastructure (elog!) to share it.
+ * Read 'count' bytes from WAL into 'buf', starting at location 'startptr'
+ * in timeline 'tli'.
+ *
+ * Will open, and keep open, one WAL segment stored in the static file
+ * descriptor 'sendFile'. This means if XLogRead is used once, there will
+ * always be one descriptor left open until the process ends, but never
+ * more than one.
+ *
+ * XXX This is very similar to pg_xlogdump's XLogDumpXLogRead and to XLogRead
+ * in walsender.c but for small differences (such as lack of elog() in
+ * frontend). Probably these should be merged at some point.
*/
static void
XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count)
@@ -648,8 +658,10 @@ XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count)
XLogRecPtr recptr;
Size nbytes;
+ /* Cached state across calls. */
static int sendFile = -1;
static XLogSegNo sendSegNo = 0;
+ static TimeLineID sendTLI = 0;
static uint32 sendOff = 0;
p = buf;
@@ -664,11 +676,12 @@ XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count)
startoff = recptr % XLogSegSize;
- if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo))
+ /* Do we need to switch to a different xlog segment? */
+ if (sendFile < 0 || !XLByteInSeg(recptr, sendSegNo) ||
+ sendTLI != tli)
{
char path[MAXPGPATH];
- /* Switch to another logfile segment */
if (sendFile >= 0)
close(sendFile);
@@ -692,6 +705,7 @@ XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count)
path)));
}
sendOff = 0;
+ sendTLI = tli;
}
/* Need to seek in the file? */
@@ -740,12 +754,151 @@ XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count)
}
/*
- * read_page callback for reading local xlog files
+ * Determine XLogReaderState->currTLI and ->currTLIValidUntil;
+ * XLogReadState->EndRecPtr, ->currRecPtr and ThisTimeLineID affect the
+ * decision. This may later be used to determine which xlog segment file to
+ * open, etc.
+ *
+ * We switch to an xlog segment from the new timeline eagerly when on a
+ * historical timeline, as soon as we reach the start of the xlog segment
+ * containing the timeline switch. The server copied the segment to the new
+ * timeline so all the data up to the switch point is the same, but there's no
+ * guarantee the old segment will still exist. It may have been deleted or
+ * renamed with a .partial suffix so we can't necessarily keep reading from
+ * the old TLI even though tliSwitchPoint says it's OK.
+ *
+ * Because of this, callers MAY NOT assume that currTLI is the timeline that
+ * will be in a page's xlp_tli; the page may begin on an older timeline or we
+ * might be reading from historical timeline data on a segment that's been
+ * copied to a new timeline.
+ */
+static void
+XLogReadDetermineTimeline(XLogReaderState *state)
+{
+ /* Read the history on first time through */
+ if (state->timelineHistory == NIL)
+ state->timelineHistory = readTimeLineHistory(ThisTimeLineID);
+
+ /*
+ * Are we reading the record immediately following the one we read last
+ * time? If not, then don't use the cached timeline info.
+ */
+ if (state->currRecPtr != state->EndRecPtr)
+ {
+ state->currTLI = 0;
+ state->currTLIValidUntil = InvalidXLogRecPtr;
+ }
+
+ /*
+ * Are we reading a timeline that used to be the latest one, but became
+ * historical? This can happen in a replica that gets promoted, and in a
+ * cascading replica whose upstream gets promoted. In either case,
+ * re-read the timeline history data. We cannot read past the timeline
+ * switch point, because either the records in the old timeline might be
+ * invalid, or worse, they may valid but *different* from the ones we
+ * should be reading.
+ */
+ if (state->currTLIValidUntil == InvalidXLogRecPtr &&
+ state->currTLI != ThisTimeLineID &&
+ state->currTLI != 0)
+ {
+ /* re-read timeline history */
+ list_free_deep(state->timelineHistory);
+ state->timelineHistory = readTimeLineHistory(ThisTimeLineID);
+
+ elog(DEBUG2, "timeline %u became historical during decoding",
+ state->currTLI);
+
+ /* then invalidate the cached timeline info */
+ state->currTLI = 0;
+ state->currTLIValidUntil = InvalidXLogRecPtr;
+ }
+
+ /*
+ * Are we reading a record immediately following a timeline switch? If
+ * so, we must follow the switch too.
+ */
+ if (state->currRecPtr == state->EndRecPtr &&
+ state->currTLI != 0 &&
+ state->currTLIValidUntil != InvalidXLogRecPtr &&
+ state->currRecPtr >= state->currTLIValidUntil)
+ {
+ elog(DEBUG2,
+ "requested record %X/%X is on segment containing end of TLI %u valid until %X/%X, switching to next timeline",
+ (uint32) (state->currRecPtr >> 32),
+ (uint32) state->currRecPtr,
+ state->currTLI,
+ (uint32) (state->currTLIValidUntil >> 32),
+ (uint32) (state->currTLIValidUntil));
+
+ /* invalidate TLI info so we look up the next TLI */
+ state->currTLI = 0;
+ state->currTLIValidUntil = InvalidXLogRecPtr;
+ }
+
+ if (state->currTLI == 0)
+ {
+ /*
+ * Something changed; work out what timeline this record is on. We
+ * might read it from the segment on this TLI or, if the segment is
+ * also contained by newer timelines, the copy from a newer TLI.
+ */
+ state->currTLI = tliOfPointInHistory(state->currRecPtr,
+ state->timelineHistory);
+
+ /*
+ * Look for the most recent timeline that's on the same xlog segment
+ * as this record, since that's the only one we can assume is still
+ * readable.
+ */
+ while (state->currTLI != ThisTimeLineID &&
+ state->currTLIValidUntil == InvalidXLogRecPtr)
+ {
+ XLogRecPtr tliSwitch;
+ TimeLineID nextTLI;
+
+ tliSwitch = tliSwitchPoint(state->currTLI, state->timelineHistory,
+ &nextTLI);
+
+ /* round ValidUntil down to start of seg containing the switch */
+ state->currTLIValidUntil =
+ ((tliSwitch / XLogSegSize) * XLogSegSize);
+
+ if (state->currRecPtr >= state->currTLIValidUntil)
+ {
+ /*
+ * The new currTLI ends on this WAL segment so check the next
+ * TLI to see if it's the last one on the segment.
+ *
+ * If that's the current TLI we'll stop searching.
+ */
+ state->currTLI = nextTLI;
+ state->currTLIValidUntil = InvalidXLogRecPtr;
+ }
+ }
+
+ /*
+ * We're now either reading from the first xlog segment in the current
+ * server's timeline or the most recent historical timeline that
+ * exists on the target segment.
+ */
+ elog(DEBUG2, "XLog read ptr %X/%X is on segment with TLI %u valid until %X/%X, server current TLI is %u",
+ (uint32) (state->currRecPtr >> 32),
+ (uint32) state->currRecPtr,
+ state->currTLI,
+ (uint32) (state->currTLIValidUntil >> 32),
+ (uint32) (state->currTLIValidUntil),
+ ThisTimeLineID);
+ }
+}
+
+/*
+ * XLogPageReadCB callback for reading local xlog files
*
* Public because it would likely be very helpful for someone writing another
* output method outside walsender, e.g. in a bgworker.
*
- * TODO: The walsender has it's own version of this, but it relies on the
+ * TODO: The walsender has its own version of this, but it relies on the
* walsender's latch being set whenever WAL is flushed. No such infrastructure
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
@@ -754,46 +907,99 @@ int
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
- XLogRecPtr flushptr,
+ XLogRecPtr read_upto,
loc;
int count;
loc = targetPagePtr + reqLen;
+
+ /* Make sure enough xlog is available... */
while (1)
{
/*
- * TODO: we're going to have to do something more intelligent about
- * timelines on standbys. Use readTimeLineHistory() and
- * tliOfPointInHistory() to get the proper LSN? For now we'll catch
- * that case earlier, but the code and TODO is left in here for when
- * that changes.
+ * Check which timeline to get the record from.
+ *
+ * We have to do it each time through the loop because if we're in
+ * recovery as a cascading standby, the current timeline might've
+ * become historical.
*/
- if (!RecoveryInProgress())
+ XLogReadDetermineTimeline(state);
+
+ if (state->currTLI == ThisTimeLineID)
{
- *pageTLI = ThisTimeLineID;
- flushptr = GetFlushRecPtr();
+ /*
+ * We're reading from the current timeline so we might have to
+ * wait for the desired record to be generated (or, for a standby,
+ * received & replayed)
+ */
+ if (!RecoveryInProgress())
+ {
+ *pageTLI = ThisTimeLineID;
+ read_upto = GetFlushRecPtr();
+ }
+ else
+ read_upto = GetXLogReplayRecPtr(pageTLI);
+
+ if (loc <= read_upto)
+ break;
+
+ CHECK_FOR_INTERRUPTS();
+ pg_usleep(1000L);
}
else
- flushptr = GetXLogReplayRecPtr(pageTLI);
-
- if (loc <= flushptr)
+ {
+ /*
+ * We're on a historical timeline, so limit reading to the switch
+ * point where we moved to the next timeline.
+ *
+ * We don't need to GetFlushRecPtr or GetXLogReplayRecPtr. We know
+ * about the new timeline, so we must've received past the end of
+ * it.
+ */
+ read_upto = state->currTLIValidUntil;
+
+ /*
+ * Setting pageTLI to our wanted record's TLI is slightly wrong;
+ * the page might begin on an older timeline if it contains a
+ * timeline switch, since its xlog segment will have been copied
+ * from the prior timeline. This is pretty harmless though, as
+ * nothing cares so long as the timeline doesn't go backwards. We
+ * should read the page header instead; FIXME someday.
+ */
+ *pageTLI = state->currTLI;
+
+ /* No need to wait on a historical timeline */
break;
-
- CHECK_FOR_INTERRUPTS();
- pg_usleep(1000L);
+ }
}
- /* more than one block available */
- if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
+ if (targetPagePtr + XLOG_BLCKSZ <= read_upto)
+ {
+ /*
+ * more than one block available; read only that block, have caller
+ * come back if they need more.
+ */
count = XLOG_BLCKSZ;
- /* not enough data there */
- else if (targetPagePtr + reqLen > flushptr)
+ }
+ else if (targetPagePtr + reqLen > read_upto)
+ {
+ /* not enough data there */
return -1;
- /* part of the page available */
+ }
else
- count = flushptr - targetPagePtr;
+ {
+ /* enough bytes available to satisfy the request */
+ count = read_upto - targetPagePtr;
+ }
+ /*
+ * Even though we just determined how much of the page can be
+ * validly read as 'count', read the whole page anyway. It's
+ * guaranteed to be zero-padded out the page boundary if it's
+ * incomplete.
+ */
XLogRead(cur_page, *pageTLI, targetPagePtr, XLOG_BLCKSZ);
+ /* number of valid bytes in the buffer */
return count;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index f789fc1..83e5b9e 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -115,7 +115,7 @@ logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ targetRecPtr, cur_page, pageTLI);
}
/*
@@ -231,16 +231,14 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
rsinfo->setResult = p->tupstore;
rsinfo->setDesc = p->tupdesc;
- /* compute the current end-of-wal */
- if (!RecoveryInProgress())
- end_of_wal = GetFlushRecPtr();
- else
- end_of_wal = GetXLogReplayRecPtr(NULL);
-
ReplicationSlotAcquire(NameStr(*name));
PG_TRY();
{
+ /*
+ * Passing InvalidXLogRecPtr here causes replay to users to start at
+ * the confirmed_lsn on the slot.
+ */
ctx = CreateDecodingContext(InvalidXLogRecPtr,
options,
logical_read_local_xlog_page,
@@ -263,6 +261,15 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
ctx->output_writer_private = p;
+ /*
+ * We start reading xlog from the restart lsn, even though in
+ * CreateDecodingContext we set the snapshot builder up using the
+ * slot's confirmed_flush. This means we might read xlog we don't
+ * actually decode rows from, but the snapshot builder might need it to
+ * get to a consistent point. The point we start returning data to
+ * *users* at is the confirmed_flush lsn set up in the decoding
+ * context.
+ */
startptr = MyReplicationSlot->data.restart_lsn;
CurrentResourceOwner = ResourceOwnerCreate(CurrentResourceOwner, "logical decoding");
@@ -270,8 +277,14 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
/* invalidate non-timetravel entries */
InvalidateSystemCaches();
+ if (!RecoveryInProgress())
+ end_of_wal = GetFlushRecPtr();
+ else
+ end_of_wal = GetXLogReplayRecPtr(NULL);
+
+ /* Decode until we run out of records */
while ((startptr != InvalidXLogRecPtr && startptr < end_of_wal) ||
- (ctx->reader->EndRecPtr && ctx->reader->EndRecPtr < end_of_wal))
+ (ctx->reader->EndRecPtr != InvalidXLogRecPtr && ctx->reader->EndRecPtr < end_of_wal))
{
XLogRecord *record;
char *errm = NULL;
@@ -280,6 +293,10 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
if (errm)
elog(ERROR, "%s", errm);
+ /*
+ * Now that we've set up the xlog reader state subsequent calls
+ * pass InvalidXLogRecPtr to say "continue from last record"
+ */
startptr = InvalidXLogRecPtr;
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 7553cc4..31cafd3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -27,6 +27,10 @@
#include "access/xlogrecord.h"
+#ifndef FRONTEND
+#include "nodes/pg_list.h"
+#endif
+
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
@@ -139,26 +143,50 @@ struct XLogReaderState
* ----------------------------------------
*/
- /* Buffer for currently read page (XLOG_BLCKSZ bytes) */
+ /*
+ * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
+ * readLen bytes)
+ */
char *readBuf;
- /* last read segment, segment offset, read length, TLI */
+ /*
+ * last read segment, segment offset, read length, TLI for data currently
+ * in readBuf.
+ */
XLogSegNo readSegNo;
uint32 readOff;
uint32 readLen;
TimeLineID readPageTLI;
- /* beginning of last page read, and its TLI */
+ /*
+ * beginning of prior page read, and its TLI. Doesn't necessarily
+ * correspond to what's in readBuf, used for timeline sanity checks.
+ */
XLogRecPtr latestPagePtr;
TimeLineID latestPageTLI;
/* beginning of the WAL record being read. */
XLogRecPtr currRecPtr;
+ /* timeline to read it from, 0 if a lookup is required */
+ TimeLineID currTLI;
+
+ /*
+ * Pointer to the end of the last whole segment on the timeline in currTLI
+ * if it's historical or InvalidXLogRecPtr if currTLI is the current
+ * timeline. This is *not* the tliSwitchPoint but it's guaranteed safe to
+ * read up to this point from currTLI.
+ */
+ XLogRecPtr currTLIValidUntil;
/* Buffer for current ReadRecord result (expandable) */
char *readRecordBuf;
uint32 readRecordBufSize;
+#ifndef FRONTEND
+ /* cached timeline history, only available in backend */
+ List *timelineHistory;
+#endif
+
/* Buffer to hold error message */
char *errormsg_buf;
};
@@ -174,6 +202,9 @@ extern void XLogReaderFree(XLogReaderState *state);
extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
XLogRecPtr recptr, char **errormsg);
+/* Flush any cached page */
+extern void XLogReaderInvalCache(XLogReaderState *state);
+
#ifdef FRONTEND
extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
#endif /* FRONTEND */
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 1b9abce..c9df35e 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -48,6 +48,6 @@ extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
extern int read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
- int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI);
+ int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI);
#endif
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 6167ec1..ebdcdc8 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -13,6 +13,7 @@ SUBDIRS = \
test_parser \
test_rls_hooks \
test_shm_mq \
+ test_slot_timelines \
worker_spi
all: submake-errcodes
diff --git a/src/test/modules/test_slot_timelines/.gitignore b/src/test/modules/test_slot_timelines/.gitignore
new file mode 100644
index 0000000..543c50d
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/.gitignore
@@ -0,0 +1,3 @@
+results/
+tmp_check/
+log/
diff --git a/src/test/modules/test_slot_timelines/Makefile b/src/test/modules/test_slot_timelines/Makefile
new file mode 100644
index 0000000..21757c5
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/Makefile
@@ -0,0 +1,22 @@
+# src/test/modules/test_slot_timelines/Makefile
+
+MODULES = test_slot_timelines
+PGFILEDESC = "test_slot_timelines - test utility for slot timeline following"
+
+EXTENSION = test_slot_timelines
+DATA = test_slot_timelines--1.0.sql
+
+EXTRA_INSTALL=contrib/test_decoding
+REGRESS=load_extension
+REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/test_slot_timelines/test_slot_timelines.conf
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_slot_timelines
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_slot_timelines/README b/src/test/modules/test_slot_timelines/README
new file mode 100644
index 0000000..585f02f
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/README
@@ -0,0 +1,19 @@
+A test module for logical decoding failover and timeline following.
+
+This module provides a minimal way to maintain logical slots on replicas
+that mirror the state on the master. It doesn't make decoding possible,
+just tracking slot state so that a decoding client that's using the master
+can follow a physical failover to the standby. The master doesn't know
+about the slots on the standby, they're synced by a client that connects
+to both.
+
+This is intentionally not part of the test_decoding module because that's meant
+to serve as example code, where this module exercises internal server features
+by unsafely exposing internal state to SQL. It's not the right way to do
+failover, it's just a simple way to test it from the perl TAP framework to
+prove the feature works.
+
+In a practical implementation of this approach a bgworker on the master would
+monitor slot positions and relay them to a bgworker on the standby that applies
+the position updates without exposing slot internals to SQL. That's too complex
+for this test framework though.
diff --git a/src/test/modules/test_slot_timelines/expected/load_extension.out b/src/test/modules/test_slot_timelines/expected/load_extension.out
new file mode 100644
index 0000000..14a414a
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/expected/load_extension.out
@@ -0,0 +1,19 @@
+CREATE EXTENSION test_slot_timelines;
+SELECT test_slot_timelines_create_logical_slot('test_slot', 'test_decoding');
+ test_slot_timelines_create_logical_slot
+-----------------------------------------
+
+(1 row)
+
+SELECT test_slot_timelines_advance_logical_slot('test_slot', txid_current(), txid_current(), pg_current_xlog_location(), pg_current_xlog_location());
+ test_slot_timelines_advance_logical_slot
+------------------------------------------
+
+(1 row)
+
+SELECT pg_drop_replication_slot('test_slot');
+ pg_drop_replication_slot
+--------------------------
+
+(1 row)
+
diff --git a/src/test/modules/test_slot_timelines/sql/load_extension.sql b/src/test/modules/test_slot_timelines/sql/load_extension.sql
new file mode 100644
index 0000000..a71127d
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/sql/load_extension.sql
@@ -0,0 +1,7 @@
+CREATE EXTENSION test_slot_timelines;
+
+SELECT test_slot_timelines_create_logical_slot('test_slot', 'test_decoding');
+
+SELECT test_slot_timelines_advance_logical_slot('test_slot', txid_current(), txid_current(), pg_current_xlog_location(), pg_current_xlog_location());
+
+SELECT pg_drop_replication_slot('test_slot');
diff --git a/src/test/modules/test_slot_timelines/test_slot_timelines--1.0.sql b/src/test/modules/test_slot_timelines/test_slot_timelines--1.0.sql
new file mode 100644
index 0000000..31d7f8e
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/test_slot_timelines--1.0.sql
@@ -0,0 +1,16 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_slot_timelines" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION test_slot_timelines_create_logical_slot(slot_name text, plugin text)
+RETURNS void
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+COMMENT ON FUNCTION test_slot_timelines_create_logical_slot(text, text)
+IS 'Create a logical slot at a particular lsn and xid. Do not use in production servers, it is not safe. The slot is created with an invalid xmin and lsn.';
+
+CREATE OR REPLACE FUNCTION test_slot_timelines_advance_logical_slot(slot_name text, new_xmin bigint, new_catalog_xmin bigint, new_restart_lsn pg_lsn, new_confirmed_lsn pg_lsn)
+RETURNS void
+LANGUAGE c AS 'MODULE_PATHNAME';
+
+COMMENT ON FUNCTION test_slot_timelines_advance_logical_slot(text, bigint, bigint, pg_lsn, pg_lsn)
+IS 'Advance a logical slot directly. Do not use this in production servers, it is not safe.';
diff --git a/src/test/modules/test_slot_timelines/test_slot_timelines.c b/src/test/modules/test_slot_timelines/test_slot_timelines.c
new file mode 100644
index 0000000..74dd1a0
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/test_slot_timelines.c
@@ -0,0 +1,133 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_slot_timelines.c
+ * Test harness code for slot timeline following
+ *
+ * Copyright (c) 2016, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_slot_timelines/test_slot_timelines.c
+ *
+ * -------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/transam.h"
+#include "fmgr.h"
+#include "miscadmin.h"
+#include "replication/slot.h"
+#include "utils/builtins.h"
+#include "utils/pg_lsn.h"
+
+PG_MODULE_MAGIC;
+
+PG_FUNCTION_INFO_V1(test_slot_timelines_create_logical_slot);
+PG_FUNCTION_INFO_V1(test_slot_timelines_advance_logical_slot);
+
+static void clear_slot_transient_state(void);
+
+/*
+ * Create a new logical slot, with invalid LSN and xid, directly. This does not
+ * use the snapshot builder or logical decoding machinery. It's only intended
+ * for creating a slot on a replica that mirrors the state of a slot on an
+ * upstream master.
+ *
+ * Note that this is test harness code. You shouldn't expose slot internals
+ * to SQL like this for any real world usage. See the README.
+ */
+Datum
+test_slot_timelines_create_logical_slot(PG_FUNCTION_ARGS)
+{
+ char *slotname = text_to_cstring(PG_GETARG_TEXT_P(0));
+ char *plugin = text_to_cstring(PG_GETARG_TEXT_P(1));
+
+ CheckSlotRequirements();
+
+ ReplicationSlotCreate(slotname, true, RS_PERSISTENT);
+
+ /* register the plugin name with the slot */
+ StrNCpy(NameStr(MyReplicationSlot->data.plugin), plugin, NAMEDATALEN);
+
+ /*
+ * Initialize persistent state to placeholders to be set by
+ * test_slot_timelines_advance_logical_slot .
+ */
+ MyReplicationSlot->data.xmin = InvalidTransactionId;
+ MyReplicationSlot->data.catalog_xmin = InvalidTransactionId;
+ MyReplicationSlot->data.restart_lsn = InvalidXLogRecPtr;
+ MyReplicationSlot->data.confirmed_flush = InvalidXLogRecPtr;
+
+ clear_slot_transient_state();
+
+ ReplicationSlotRelease();
+
+ PG_RETURN_VOID();
+}
+
+/*
+ * Set the state of a slot.
+ *
+ * This doesn't maintain the non-persistent state at all,
+ * but since the slot isn't in use that's OK.
+ *
+ * There's intentionally no check to prevent slots going backwards
+ * because they can actually go backwards if the master crashes when
+ * it hasn't yet flushed slot state to disk then we copy the older
+ * slot state after recovery.
+ *
+ * There's no checking done for xmin or catalog xmin either, since
+ * we can't really do anything useful that accounts for xid wrap-around.
+ *
+ * Note that this is test harness code. You shouldn't expose slot internals
+ * to SQL like this for any real world usage. See the README.
+ */
+Datum
+test_slot_timelines_advance_logical_slot(PG_FUNCTION_ARGS)
+{
+ char *slotname = text_to_cstring(PG_GETARG_TEXT_P(0));
+ TransactionId new_xmin = (TransactionId) PG_GETARG_INT64(1);
+ TransactionId new_catalog_xmin = (TransactionId) PG_GETARG_INT64(2);
+ XLogRecPtr restart_lsn = PG_GETARG_LSN(3);
+ XLogRecPtr confirmed_lsn = PG_GETARG_LSN(4);
+
+ CheckSlotRequirements();
+
+ ReplicationSlotAcquire(slotname);
+
+ if (MyReplicationSlot->data.database != MyDatabaseId)
+ elog(ERROR, "Trying to update a slot on a different database");
+
+ MyReplicationSlot->data.xmin = new_xmin;
+ MyReplicationSlot->data.catalog_xmin = new_catalog_xmin;
+ MyReplicationSlot->data.restart_lsn = restart_lsn;
+ MyReplicationSlot->data.confirmed_flush = confirmed_lsn;
+
+ clear_slot_transient_state();
+
+ ReplicationSlotMarkDirty();
+ ReplicationSlotSave();
+ ReplicationSlotRelease();
+
+ ReplicationSlotsComputeRequiredXmin(false);
+ ReplicationSlotsComputeRequiredLSN();
+
+ PG_RETURN_VOID();
+}
+
+static void
+clear_slot_transient_state(void)
+{
+ Assert(MyReplicationSlot != NULL);
+
+ /*
+ * Make sure the slot state is the same as if it were newly loaded from
+ * disk on recovery.
+ */
+ MyReplicationSlot->effective_xmin = MyReplicationSlot->data.xmin;
+ MyReplicationSlot->effective_catalog_xmin = MyReplicationSlot->data.catalog_xmin;
+
+ MyReplicationSlot->candidate_catalog_xmin = InvalidTransactionId;
+ MyReplicationSlot->candidate_xmin_lsn = InvalidXLogRecPtr;
+ MyReplicationSlot->candidate_restart_lsn = InvalidXLogRecPtr;
+ MyReplicationSlot->candidate_restart_valid = InvalidXLogRecPtr;
+}
diff --git a/src/test/modules/test_slot_timelines/test_slot_timelines.conf b/src/test/modules/test_slot_timelines/test_slot_timelines.conf
new file mode 100644
index 0000000..56b46d7
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/test_slot_timelines.conf
@@ -0,0 +1,2 @@
+max_replication_slots=2
+wal_level=logical
diff --git a/src/test/modules/test_slot_timelines/test_slot_timelines.control b/src/test/modules/test_slot_timelines/test_slot_timelines.control
new file mode 100644
index 0000000..dcee1a7
--- /dev/null
+++ b/src/test/modules/test_slot_timelines/test_slot_timelines.control
@@ -0,0 +1,5 @@
+# test_slot_timelines extension
+comment = 'Test utility for slot timeline following and logical decoding'
+default_version = '1.0'
+module_pathname = '$libdir/test_slot_timelines'
+relocatable = true
diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index 9290719..78570dd 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,6 +9,8 @@
#
#-------------------------------------------------------------------------
+EXTRA_INSTALL=contrib/test_decoding src/test/modules/test_slot_timelines
+
subdir = src/test/recovery
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
diff --git a/src/test/recovery/t/006_logical_decoding_timelines.pl b/src/test/recovery/t/006_logical_decoding_timelines.pl
new file mode 100644
index 0000000..bc20f40
--- /dev/null
+++ b/src/test/recovery/t/006_logical_decoding_timelines.pl
@@ -0,0 +1,304 @@
+# Demonstrate that logical can follow timeline switches.
+#
+# Logical replication slots can follow timeline switches but it's
+# normally not possible to have a logical slot on a replica where
+# promotion and a timeline switch can occur. The only ways
+# we can create that circumstance are:
+#
+# * By doing a filesystem-level copy of the DB, since pg_basebackup
+# excludes pg_replslot but we can copy it directly; or
+#
+# * by creating a slot directly at the C level on the replica and
+# advancing it as we go using the low level APIs. It can't be done
+# from SQL since logical decoding isn't allowed on replicas.
+#
+# This module uses the first approach to show that timeline following
+# on a logical slot works.
+#
+use strict;
+use warnings;
+
+use PostgresNode;
+use TestLib;
+use Test::More tests => 20;
+use RecursiveCopy;
+use File::Copy;
+
+my ($stdout, $stderr, $ret);
+
+# Initialize master node
+my $node_master = get_new_node('master');
+$node_master->init(allows_streaming => 1, has_archiving => 1);
+$node_master->append_conf('postgresql.conf', "wal_level = 'logical'\n");
+$node_master->append_conf('postgresql.conf', "max_replication_slots = 2\n");
+$node_master->append_conf('postgresql.conf', "max_wal_senders = 2\n");
+$node_master->append_conf('postgresql.conf', "log_min_messages = 'debug2'\n");
+$node_master->dump_info;
+$node_master->start;
+
+diag "Testing logical timeline following with a filesystem-level copy";
+
+$node_master->safe_psql('postgres',
+"SELECT pg_create_logical_replication_slot('before_basebackup', 'test_decoding');"
+);
+$node_master->safe_psql('postgres', "CREATE TABLE decoding(blah text);");
+$node_master->safe_psql('postgres',
+ "INSERT INTO decoding(blah) VALUES ('beforebb');");
+$node_master->safe_psql('postgres', 'CHECKPOINT;');
+
+my $backup_name = 'b1';
+$node_master->backup_fs_hot($backup_name);
+
+my $node_replica = get_new_node('replica');
+$node_replica->init_from_backup(
+ $node_master, $backup_name,
+ has_streaming => 1,
+ has_restoring => 1);
+$node_replica->start;
+
+$node_master->safe_psql('postgres',
+"SELECT pg_create_logical_replication_slot('after_basebackup', 'test_decoding');"
+);
+$node_master->safe_psql('postgres',
+ "INSERT INTO decoding(blah) VALUES ('afterbb');");
+$node_master->safe_psql('postgres', 'CHECKPOINT;');
+
+# Verify that only the before base_backup slot is on the replica
+$stdout = $node_replica->safe_psql('postgres',
+ 'SELECT slot_name FROM pg_replication_slots ORDER BY slot_name');
+is($stdout, 'before_basebackup',
+ 'Expected to find only slot before_basebackup on replica');
+
+# Boom, crash
+$node_master->stop('immediate');
+
+$node_replica->promote;
+$node_replica->poll_query_until('postgres',
+ "SELECT NOT pg_is_in_recovery();");
+
+$node_replica->safe_psql('postgres',
+ "INSERT INTO decoding(blah) VALUES ('after failover');");
+
+# Shouldn't be able to read from slot created after base backup
+($ret, $stdout, $stderr) = $node_replica->psql('postgres',
+"SELECT data FROM pg_logical_slot_peek_changes('after_basebackup', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');"
+);
+is($ret, 3, 'replaying from after_basebackup slot fails');
+like(
+ $stderr,
+ qr/replication slot "after_basebackup" does not exist/,
+ 'after_basebackup slot missing');
+
+# Should be able to read from slot created before base backup
+($ret, $stdout, $stderr) = $node_replica->psql(
+ 'postgres',
+"SELECT data FROM pg_logical_slot_peek_changes('before_basebackup', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');",
+ timeout => 30);
+is($ret, 0, 'replay from slot before_basebackup succeeds');
+is( $stdout, q(BEGIN
+table public.decoding: INSERT: blah[text]:'beforebb'
+COMMIT
+BEGIN
+table public.decoding: INSERT: blah[text]:'afterbb'
+COMMIT
+BEGIN
+table public.decoding: INSERT: blah[text]:'after failover'
+COMMIT), 'decoded expected data from slot before_basebackup');
+is($stderr, '', 'replay from slot before_basebackup produces no stderr');
+
+# We don't need the standby anymore
+$node_replica->teardown_node();
+
+
+# OK, time to try the same thing again, but this time we'll be using slot
+# mirroring on the standby and a pg_basebackup of the master.
+
+diag "Testing logical timeline following with test_slot_timelines module";
+
+$node_master->start();
+
+# Clean up after the last test
+$node_master->safe_psql('postgres', 'DELETE FROM decoding;');
+is( $node_master->psql(
+ 'postgres',
+'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots;'),
+ 0,
+ 'dropping slots succeeds via pg_drop_replication_slot');
+
+# Same as before, we'll make one slot before basebackup, one after. This time
+# the basebackup will be with pg_basebackup so it'll omit both slots, then
+# we'll use SQL functions provided by the test_slot_timelines test module to sync
+# them to the replica, do some work, sync them and fail over then test again.
+# This time we should have both the before- and after-basebackup slots working.
+
+is( $node_master->psql(
+ 'postgres',
+"SELECT pg_create_logical_replication_slot('before_basebackup', 'test_decoding');"
+ ),
+ 0,
+ 'creating slot before_basebackup succeeds');
+
+$node_master->safe_psql('postgres',
+ "INSERT INTO decoding(blah) VALUES ('beforebb');");
+
+$backup_name = 'b2';
+$node_master->backup($backup_name);
+
+is( $node_master->psql(
+ 'postgres',
+"SELECT pg_create_logical_replication_slot('after_basebackup', 'test_decoding');"
+ ),
+ 0,
+ 'creating slot after_basebackup succeeds');
+
+$node_master->safe_psql('postgres',
+ "INSERT INTO decoding(blah) VALUES ('afterbb');");
+
+$node_replica = get_new_node('replica2');
+$node_replica->init_from_backup(
+ $node_master, $backup_name,
+ has_streaming => 1,
+ has_restoring => 1);
+
+$node_replica->start;
+
+# Verify the slots are both absent on the replica
+$stdout = $node_replica->safe_psql('postgres',
+ 'SELECT slot_name FROM pg_replication_slots ORDER BY slot_name');
+is($stdout, '', 'No slots exist on the replica');
+
+# Now do our magic to sync the slot states across. Normally
+# this would be being done continuously by a bgworker but
+# we're just doing it by hand for this test. This is exposing
+# postgres innards to SQL so it's unsafe except for testing.
+$node_master->safe_psql('postgres', 'CREATE EXTENSION test_slot_timelines;');
+my $slotinfo = $node_master->safe_psql('postgres',
+'SELECT slot_name, plugin, xmin, catalog_xmin, restart_lsn, confirmed_flush_lsn FROM pg_replication_slots ORDER BY slot_name'
+);
+diag "Copying slots to replica";
+open my $fh, '<', \$slotinfo or die $!;
+while (<$fh>)
+{
+ print $_;
+ chomp $_;
+ my ($slot_name, $plugin, $xmin, $catalog_xmin, $restart_lsn,
+ $confirmed_flush_lsn)
+ = map {
+ if ($_ ne '') { "'$_'" }
+ else { 'NULL' }
+ } split qr/\|/, $_;
+
+ print
+"# Copying slot $slot_name,$plugin,$xmin,$catalog_xmin,$restart_lsn,$confirmed_flush_lsn\n";
+ $node_replica->safe_psql('postgres',
+ "SELECT test_slot_timelines_create_logical_slot($slot_name, $plugin);"
+ );
+ $node_replica->safe_psql('postgres',
+"SELECT test_slot_timelines_advance_logical_slot($slot_name, $xmin, $catalog_xmin, $restart_lsn, $confirmed_flush_lsn);"
+ );
+}
+close $fh or die $!;
+
+# Now both slots are present on the replica and exactly match the master
+$stdout = $node_replica->safe_psql('postgres',
+ 'SELECT slot_name FROM pg_replication_slots ORDER BY slot_name');
+is( $stdout,
+ "after_basebackup\nbefore_basebackup",
+ 'both slots now exist on replica');
+
+$stdout = $node_replica->safe_psql(
+ 'postgres',
+ qq{SELECT slot_name, plugin, xmin, catalog_xmin,
+ restart_lsn, confirmed_flush_lsn
+ FROM pg_replication_slots
+ ORDER BY slot_name});
+is($stdout, $slotinfo,
+ "slot data read back from replica matches slot data on master");
+
+# We now have to copy some extra WAL to satisfy the requirements of the oldest
+# replication slot. pg_basebackup doesn't know to copy the extra WAL for slots
+# so we have to help out. We know the WAL is still retained on the master
+# because we haven't advanced the slots there.
+#
+# Figure out what the oldest segment we need is by looking at the restart_lsn
+# of the oldest slot.
+#
+# It only makes sense to do this once the slots are created on the replica,
+# otherwise it might just delete the segments again.
+
+my $oldest_needed_segment = $node_master->safe_psql(
+ 'postgres',
+ qq{SELECT pg_xlogfile_name((
+ SELECT restart_lsn
+ FROM pg_replication_slots
+ ORDER BY restart_lsn ASC
+ LIMIT 1
+ ));}
+);
+
+diag "oldest needed xlog seg is $oldest_needed_segment ";
+
+# WAL segment names sort lexically so we can just grab everything > than this
+# segment.
+opendir(my $pg_xlog, $node_master->data_dir . "/pg_xlog") or die $!;
+while (my $seg = readdir $pg_xlog)
+{
+ next unless $seg >= $oldest_needed_segment && $seg =~ /^[0-9]{24}/;
+ diag "copying xlog seg $seg";
+ copy(
+ $node_master->data_dir . "/pg_xlog/" . $seg,
+ $node_replica->data_dir . "/pg_xlog/" . $seg
+ ) or die "copy of xlog seg $seg failed: $!";
+}
+closedir $pg_xlog;
+
+# Boom, crash the master
+$node_master->stop('immediate');
+
+$node_replica->promote;
+$node_replica->poll_query_until('postgres',
+ "SELECT NOT pg_is_in_recovery();");
+
+$node_replica->safe_psql('postgres',
+ "INSERT INTO decoding(blah) VALUES ('after failover');");
+
+# This time we can read from both slots
+($ret, $stdout, $stderr) = $node_replica->psql(
+ 'postgres',
+"SELECT data FROM pg_logical_slot_peek_changes('after_basebackup', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');",
+ timeout => 30);
+is($ret, 0, 'replay from slot after_basebackup succeeds');
+is( $stdout, q(BEGIN
+table public.decoding: INSERT: blah[text]:'afterbb'
+COMMIT
+BEGIN
+table public.decoding: INSERT: blah[text]:'after failover'
+COMMIT), 'decoded expected data from slot after_basebackup');
+is($stderr, '', 'replay from slot after_basebackup produces no stderr');
+
+# Should be able to read from slot created before base backup
+#
+# This would fail with an error about missing WAL segments if we hadn't
+# copied extra WAL earlier.
+($ret, $stdout, $stderr) = $node_replica->psql(
+ 'postgres',
+"SELECT data FROM pg_logical_slot_peek_changes('before_basebackup', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');",
+ timeout => 30);
+is($ret, 0, 'replay from slot before_basebackup succeeds');
+is( $stdout, q(BEGIN
+table public.decoding: INSERT: blah[text]:'beforebb'
+COMMIT
+BEGIN
+table public.decoding: INSERT: blah[text]:'afterbb'
+COMMIT
+BEGIN
+table public.decoding: INSERT: blah[text]:'after failover'
+COMMIT), 'decoded expected data from slot before_basebackup');
+is($stderr, '', 'replay from slot before_basebackup produces no stderr');
+
+($ret, $stdout, $stderr) = $node_replica->psql('postgres',
+ 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots;');
+is($ret, 0, 'dropping slots succeeds via pg_drop_replication_slot');
+is($stderr, '', 'dropping slots produces no stderr output');
+
+1;
--
2.1.0