From 9e6d49e04bb6b4f92e0a31b92041594f379fdbf0 Mon Sep 17 00:00:00 2001 From: "houzj.fnst" Date: Wed, 20 Apr 2022 16:45:07 +0800 Subject: [PATCH v40] Perform streaming logical transactions by parallel workers Currently, for large transactions, the publisher sends the data in multiple streams (changes divided into chunks depending upon logical_decoding_work_mem), and then on the subscriber-side, the apply worker writes the changes into temporary files and once it receives the commit, it reads from those files and applies the entire transaction. To improve the performance of such transactions, we can instead allow them to be applied via parallel workers. In this approach, we assign a new parallel apply worker (if available) as soon as the xact's first stream is received and the leader apply worker will send changes to this new worker via shared memory. The parallel apply worker will directly apply the change instead of writing it to temporary files. We keep this worker assigned till the transaction commit is received and also wait for the worker to finish at commit. This preserves commit ordering and avoids writing to and reading from files in most cases. We still need to spill if there is no worker available. This patch also extends the SUBSCRIPTION 'streaming' parameter so that the user can control whether to apply the streaming transaction in a parallel apply worker or spill the change to disk. The user can set the streaming parameter to 'on/off', 'parallel'. The parameter value 'parallel' means the streaming will be applied via a parallel apply worker, if available. The parameter value 'on' means the streaming transaction will be spilled to disk. The default value is 'off' (same as current behaviour). In addition, the patch extends the logical replication STREAM_ABORT message so that abort_time and abort_lsn can also be sent which can be used to update the replication origin in parallel apply worker when the streaming transaction is aborted. Because this message extension is needed to support parallel streaming, parallel streaming is not supported for publications on servers < PG16. --- doc/src/sgml/catalogs.sgml | 11 +- doc/src/sgml/config.sgml | 28 +- doc/src/sgml/logical-replication.sgml | 21 +- doc/src/sgml/protocol.sgml | 29 +- doc/src/sgml/ref/create_subscription.sgml | 24 +- src/backend/access/transam/xact.c | 13 + src/backend/commands/define.c | 58 ++ src/backend/commands/subscriptioncmds.c | 10 +- src/backend/libpq/pqmq.c | 18 +- src/backend/postmaster/bgworker.c | 3 + .../libpqwalreceiver/libpqwalreceiver.c | 6 +- src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 1099 ++++++++++++++++++++ src/backend/replication/logical/decode.c | 10 +- src/backend/replication/logical/launcher.c | 204 +++- src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/proto.c | 37 +- src/backend/replication/logical/reorderbuffer.c | 10 +- src/backend/replication/logical/tablesync.c | 20 +- src/backend/replication/logical/worker.c | 960 +++++++++++++---- src/backend/replication/pgoutput/pgoutput.c | 21 +- src/backend/storage/ipc/procsignal.c | 4 + src/backend/tcop/postgres.c | 3 + src/backend/utils/activity/wait_event.c | 6 + src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/bin/pg_dump/pg_dump.c | 6 +- src/include/catalog/pg_subscription.h | 21 +- src/include/commands/defrem.h | 1 + src/include/replication/logicallauncher.h | 1 + src/include/replication/logicalproto.h | 28 +- src/include/replication/logicalworker.h | 8 + src/include/replication/pgoutput.h | 2 +- src/include/replication/reorderbuffer.h | 7 +- src/include/replication/walreceiver.h | 2 +- src/include/replication/worker_internal.h | 134 ++- src/include/storage/procsignal.h | 1 + src/include/utils/wait_event.h | 2 + src/test/regress/expected/subscription.out | 12 +- src/test/regress/sql/subscription.sql | 6 +- src/tools/pgindent/typedefs.list | 5 + 41 files changed, 2555 insertions(+), 291 deletions(-) create mode 100644 src/backend/replication/logical/applyparallelworker.c diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 00f833d..c5769da 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7892,11 +7892,16 @@ SCRAM-SHA-256$<iteration count>:&l - substream bool + substream char - If true, the subscription will allow streaming of in-progress - transactions + Controls how to handle the streaming of in-progress transactions: + f = disallow streaming of in-progress transactions, + t = spill the changes of in-progress transactions to + disk and apply at once after the transaction is committed on the + publisher, + p = apply changes directly using a parallel apply + worker if available (same as 't' if no worker is available) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 6c64933..c94dd0e 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4948,7 +4948,8 @@ ANY num_sync ( num_sync ( + max_parallel_apply_workers_per_subscription (integer) + + max_parallel_apply_workers_per_subscription configuration parameter + + + + + Maximum number of parallel apply workers per subscription. This + parameter controls the amount of parallelism for streaming of + in-progress transactions with subscription parameter + streaming = parallel. + + + The parallel apply workers are taken from the pool defined by + max_logical_replication_workers. + + + The default value is 2. This parameter can only be set in the + postgresql.conf file or on the server command + line. + + + + diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index e98538e..9e753fe 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -1340,6 +1340,16 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER might not violate any constraint. This can easily make the subscriber inconsistent. + + + When the streaming mode is parallel, the finish LSN of + failed transactions may not be logged. In that case, it may be necessary to + change the streaming mode to on or off and + cause the same conflicts again so the finish LSN of the failed transaction will + be written to the server log. For the usage of finish LSN, please refer to ALTER SUBSCRIPTION ... + SKIP. + @@ -1521,7 +1531,8 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER subscription. A disabled subscription or a crashed subscription will have zero rows in this view. If the initial data synchronization of any table is in progress, there will be additional workers for the tables - being synchronized. + being synchronized. Moreover, if the streaming transaction is applied in + parallel, there will be additional workers. @@ -1616,8 +1627,12 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER to the subscriber, plus some reserve for table synchronization. max_logical_replication_workers must be set to at least the number of subscriptions, again plus some reserve for the table - synchronization. Additionally the max_worker_processes - may need to be adjusted to accommodate for replication workers, at least + synchronization. If the subscription parameter streaming + is set to parallel, + max_logical_replication_workers should be increased + according to the desired number of parallel apply workers. Additionally the + max_worker_processes may need to be adjusted to + accommodate for replication workers, at least (max_logical_replication_workers + 1). Note that some extensions and parallel queries also take worker slots from max_worker_processes. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 5fdd429..c6a0e80 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -3102,7 +3102,7 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" Protocol version. Currently versions 1, 2, - and 3 are supported. + 3, and 4 are supported. Version 2 is supported only for server version 14 @@ -3112,6 +3112,11 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" Version 3 is supported only for server version 15 and above, and it allows streaming of two-phase commits. + + Version 4 is supported only for server version 16 + and above, and it allows streams of large in-progress transactions to + be applied in parallel. + @@ -6882,6 +6887,28 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" + + + Int64 (XLogRecPtr) + + + The LSN of the abort. This field is available since protocol version + 4. + + + + + + Int64 (TimestampTz) + + + Abort timestamp of the transaction. The value is in number + of microseconds since PostgreSQL epoch (2000-01-01). This field is + available since protocol version 4. + + + + diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index bd12e71..03fc047 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -222,13 +222,29 @@ CREATE SUBSCRIPTION subscription_name - streaming (boolean) + streaming (enum) Specifies whether to enable streaming of in-progress transactions - for this subscription. By default, all transactions - are fully decoded on the publisher and only then sent to the - subscriber as a whole. + for this subscription. The default value is off, + meaning all transactions are fully decoded on the publisher and only + then sent to the subscriber as a whole. + + + + If set to on, the incoming changes are written to + temporary files and then applied only after the transaction is + committed on the publisher and received by the subscriber. + + + + If set to parallel, incoming changes are directly + applied via one of the parallel apply workers, if available. If no + parallel apply worker is free to handle streaming transactions then + the changes are written to temporary files and applied after the + transaction is committed. Note that if an error happens in a + parallel apply worker, the finish LSN of the remote transaction + might not be reported in the server log. diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index fd5103a..8315b93 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -1712,6 +1712,7 @@ RecordTransactionAbort(bool isSubXact) int nchildren; TransactionId *children; TimestampTz xact_time; + bool replorigin; /* * If we haven't been assigned an XID, nobody will care whether we aborted @@ -1742,6 +1743,13 @@ RecordTransactionAbort(bool isSubXact) elog(PANIC, "cannot abort transaction %u, it was already committed", xid); + /* + * Are we using the replication origins feature? Or, in other words, are + * we replaying remote actions? + */ + replorigin = (replorigin_session_origin != InvalidRepOriginId && + replorigin_session_origin != DoNotReplicateId); + /* Fetch the data we need for the abort record */ nrels = smgrGetPendingDeletes(false, &rels); nchildren = xactGetCommittedChildren(&children); @@ -1765,6 +1773,11 @@ RecordTransactionAbort(bool isSubXact) MyXactFlags, InvalidTransactionId, NULL); + if (replorigin) + /* Move LSNs forward for this replication origin */ + replorigin_session_advance(replorigin_session_origin_lsn, + XactLastRecEnd); + /* * Report the latest async abort LSN, so that the WAL writer knows to * flush this abort. There's nothing to be gained by delaying this, since diff --git a/src/backend/commands/define.c b/src/backend/commands/define.c index 86b8907..11312d1 100644 --- a/src/backend/commands/define.c +++ b/src/backend/commands/define.c @@ -36,6 +36,7 @@ #include #include "catalog/namespace.h" +#include "catalog/pg_subscription.h" #include "commands/defrem.h" #include "nodes/makefuncs.h" #include "parser/parse_type.h" @@ -346,6 +347,63 @@ defGetStringList(DefElem *def) } /* + * Extract the streaming mode value from a DefElem. This is like + * defGetBoolean() but also accepts the special value of "parallel". + */ +char +defGetStreamingMode(DefElem *def) +{ + /* + * If no parameter value given, assume "true" is meant. + */ + if (def->arg == NULL) + return SUBSTREAM_ON; + + /* + * Allow 0, 1, "false", "true", "off", "on" or "parallel". + */ + switch (nodeTag(def->arg)) + { + case T_Integer: + switch (intVal(def->arg)) + { + case 0: + return SUBSTREAM_OFF; + case 1: + return SUBSTREAM_ON; + default: + /* otherwise, error out below */ + break; + } + break; + default: + { + char *sval = defGetString(def); + + /* + * The set of strings accepted here should match up with the + * grammar's opt_boolean_or_string production. + */ + if (pg_strcasecmp(sval, "false") == 0 || + pg_strcasecmp(sval, "off") == 0) + return SUBSTREAM_OFF; + if (pg_strcasecmp(sval, "true") == 0 || + pg_strcasecmp(sval, "on") == 0) + return SUBSTREAM_ON; + if (pg_strcasecmp(sval, "parallel") == 0) + return SUBSTREAM_PARALLEL; + } + break; + } + + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s requires a Boolean value or \"parallel\"", + def->defname))); + return SUBSTREAM_OFF; /* keep compiler quiet */ +} + +/* * Raise an error about a conflicting DefElem. */ void diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index f0cec2a..5c58fe5 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -84,7 +84,7 @@ typedef struct SubOpts bool copy_data; bool refresh; bool binary; - bool streaming; + char streaming; bool twophase; bool disableonerr; char *origin; @@ -138,7 +138,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, if (IsSet(supported_opts, SUBOPT_BINARY)) opts->binary = false; if (IsSet(supported_opts, SUBOPT_STREAMING)) - opts->streaming = false; + opts->streaming = SUBSTREAM_OFF; if (IsSet(supported_opts, SUBOPT_TWOPHASE_COMMIT)) opts->twophase = false; if (IsSet(supported_opts, SUBOPT_DISABLE_ON_ERR)) @@ -241,7 +241,7 @@ parse_subscription_options(ParseState *pstate, List *stmt_options, errorConflictingDefElem(defel, pstate); opts->specified_opts |= SUBOPT_STREAMING; - opts->streaming = defGetBoolean(defel); + opts->streaming = defGetStreamingMode(defel); } else if (strcmp(defel->defname, "two_phase") == 0) { @@ -629,7 +629,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner); values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled); values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary); - values[Anum_pg_subscription_substream - 1] = BoolGetDatum(opts.streaming); + values[Anum_pg_subscription_substream - 1] = CharGetDatum(opts.streaming); values[Anum_pg_subscription_subtwophasestate - 1] = CharGetDatum(opts.twophase ? LOGICALREP_TWOPHASE_STATE_PENDING : @@ -1098,7 +1098,7 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, if (IsSet(opts.specified_opts, SUBOPT_STREAMING)) { values[Anum_pg_subscription_substream - 1] = - BoolGetDatum(opts.streaming); + CharGetDatum(opts.streaming); replaces[Anum_pg_subscription_substream - 1] = true; } diff --git a/src/backend/libpq/pqmq.c b/src/backend/libpq/pqmq.c index 4d0415e..639845f 100644 --- a/src/backend/libpq/pqmq.c +++ b/src/backend/libpq/pqmq.c @@ -13,11 +13,13 @@ #include "postgres.h" +#include "access/parallel.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "libpq/pqmq.h" #include "miscadmin.h" #include "pgstat.h" +#include "replication/logicalworker.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" @@ -162,9 +164,19 @@ mq_putmessage(char msgtype, const char *s, size_t len) result = shm_mq_sendv(pq_mq_handle, iov, 2, true, true); if (pq_mq_parallel_leader_pid != 0) - SendProcSignal(pq_mq_parallel_leader_pid, - PROCSIG_PARALLEL_MESSAGE, - pq_mq_parallel_leader_backend_id); + { + if (IsLogicalParallelApplyWorker()) + SendProcSignal(pq_mq_parallel_leader_pid, + PROCSIG_PARALLEL_APPLY_MESSAGE, + pq_mq_parallel_leader_backend_id); + else + { + Assert(IsParallelWorker()); + SendProcSignal(pq_mq_parallel_leader_pid, + PROCSIG_PARALLEL_MESSAGE, + pq_mq_parallel_leader_backend_id); + } + } if (result != SHM_MQ_WOULD_BLOCK) break; diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 0d72de2..2dfa9ab 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -128,6 +128,9 @@ static const struct }, { "ApplyWorkerMain", ApplyWorkerMain + }, + { + "ParallelApplyWorkerMain", ParallelApplyWorkerMain } }; diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 7f697b0..225da81 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -443,9 +443,9 @@ libpqrcv_startstreaming(WalReceiverConn *conn, appendStringInfo(&cmd, "proto_version '%u'", options->proto.logical.proto_version); - if (options->proto.logical.streaming && - PQserverVersion(conn->streamConn) >= 140000) - appendStringInfoString(&cmd, ", streaming 'on'"); + if (options->proto.logical.streaming_str) + appendStringInfo(&cmd, ", streaming '%s'", + options->proto.logical.streaming_str); if (options->proto.logical.twophase && PQserverVersion(conn->streamConn) >= 150000) diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index c4e2fde..2dc25e3 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(srcdir) $(CPPFLAGS) OBJS = \ + applyparallelworker.o \ decode.o \ launcher.o \ logical.o \ diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c new file mode 100644 index 0000000..328672e --- /dev/null +++ b/src/backend/replication/logical/applyparallelworker.c @@ -0,0 +1,1099 @@ +/*------------------------------------------------------------------------- + * applyparallelworker.c + * Support routines for applying xact by parallel apply worker + * + * Copyright (c) 2016-2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/applyparallelworker.c + * + * This file contains routines that are intended to support setting up, using, + * and tearing down a ParallelApplyWorkerInfo. + * + * Refer to the comments in the file header of logical/worker.c to see more + * information about parallel apply workers. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "libpq/pqformat.h" +#include "libpq/pqmq.h" +#include "mb/pg_wchar.h" +#include "pgstat.h" +#include "postmaster/interrupt.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/origin.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "storage/procarray.h" +#include "tcop/tcopprot.h" +#include "utils/inval.h" +#include "utils/memutils.h" +#include "utils/resowner.h" +#include "utils/syscache.h" + +#define PG_LOGICAL_APPLY_SHM_MAGIC 0x787ca067 + +/* + * DSM keys for parallel apply worker. Unlike other parallel execution code, + * since we don't need to worry about DSM keys conflicting with plan_node_id we + * can use small integers. + */ +#define PARALLEL_APPLY_KEY_SHARED 1 +#define PARALLEL_APPLY_KEY_MQ 2 +#define PARALLEL_APPLY_KEY_ERROR_QUEUE 3 + +/* Queue size of DSM, 16 MB for now. */ +#define DSM_QUEUE_SIZE (16 * 1024 * 1024) + +/* + * Error queue size of DSM. It is desirable to make it large enough that a + * typical ErrorResponse can be sent without blocking. That way, a worker that + * errors out can write the whole message into the queue and terminate without + * waiting for the user backend. + */ +#define DSM_ERROR_QUEUE_SIZE (16 * 1024) + +/* + * There are three fields in each message received by the parallel apply + * worker: start_lsn, end_lsn and send_time. Because we have updated these + * statistics in the leader apply worker, we can ignore these fields in the + * parallel apply worker (see function LogicalRepApplyLoop). + */ +#define SIZE_STATS_MESSAGE (2 * sizeof(XLogRecPtr) + sizeof(TimestampTz)) + +/* + * Hash table entry to map xid to the parallel apply worker state. + */ +typedef struct ParallelApplyWorkerEntry +{ + TransactionId xid; /* Hash key -- must be first */ + ParallelApplyWorkerInfo *winfo; +} ParallelApplyWorkerEntry; + +/* Parallel apply workers hash table (initialized on first use). */ +static HTAB *ParallelApplyWorkersHash = NULL; + +/* + * A list to maintain the active parallel apply workers. The information for + * the new worker is added to the list after successfully launching it. The + * list entry is removed at the end of the transaction if there are already + * enough workers in the worker pool. For more information about the worker + * pool, see comments atop worker.c. We also remove the entry from the list if + * the worker is exited due to some error. + */ +static List *ParallelApplyWorkersList = NIL; + +/* + * Information shared between leader apply worker and parallel apply worker. + */ +ParallelApplyWorkerShared *MyParallelShared = NULL; + +/* + * Is there a message pending in parallel apply worker which we need to + * receive? + */ +volatile sig_atomic_t ParallelApplyMessagePending = false; + +/* + * Cache the parallel apply worker information required for applying the + * current streaming transaction. It is used to save the cost of searching the + * hash table when applying the changes between STREAM_START and STREAM_STOP. + */ +ParallelApplyWorkerInfo *stream_apply_worker = NULL; + +/* A list to maintain subtransactions, if any. */ +List *subxactlist = NIL; + +static bool parallel_apply_can_start(TransactionId xid); +static bool parallel_apply_setup_dsm(ParallelApplyWorkerInfo *winfo); +static ParallelApplyWorkerInfo *parallel_apply_setup_worker(void); +static bool parallel_apply_get_in_xact(ParallelApplyWorkerShared *wshared); +static void parallel_apply_free_worker_info(ParallelApplyWorkerInfo *winfo); + +/* + * Returns true if it is OK to start a parallel apply worker, false otherwise. + */ +static bool +parallel_apply_can_start(TransactionId xid) +{ + if (!TransactionIdIsValid(xid)) + return false; + + /* + * Don't start a new parallel apply worker if the subscription is not using + * parallel streaming mode, or if the publisher does not support parallel + * apply. + */ + if (!MyLogicalRepWorker->parallel_apply) + return false; + + /* Only leader apply workers can start parallel apply workers. */ + if (!am_leader_apply_worker()) + return false; + + /* + * Don't start a new parallel worker if user has set skiplsn as it's + * possible that user want to skip the streaming transaction. For + * streaming transaction, we need to spill the transaction to disk so that + * we can get the last LSN of the transaction to judge whether to skip + * before starting to apply the change. + */ + if (!XLogRecPtrIsInvalid(MySubscription->skiplsn)) + return false; + + /* + * For streaming transactions that are being applied using a parallel + * apply worker, we cannot decide whether to apply the change for a + * relation that is not in the READY state (see + * should_apply_changes_for_rel) as we won't know remote_final_lsn by that + * time. So, we don't start the new parallel apply worker in this case. + */ + if (!AllTablesyncsReady()) + return false; + + return true; +} + +/* + * Start a parallel apply worker that will be used for the specified xid. + * + * If a parallel apply worker is found but not in use then re-use it, otherwise + * start a fresh one. Cache the worker information in ParallelApplyWorkersHash + * keyed by the specified xid. + */ +void +parallel_apply_start_worker(TransactionId xid) +{ + bool found; + ListCell *lc; + ParallelApplyWorkerInfo *winfo = NULL; + ParallelApplyWorkerEntry *entry; + + if (!parallel_apply_can_start(xid)) + return; + + /* First time through, initialize apply workers hashtable. */ + if (ParallelApplyWorkersHash == NULL) + { + HASHCTL ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(TransactionId); + ctl.entrysize = sizeof(ParallelApplyWorkerEntry); + ctl.hcxt = ApplyContext; + + ParallelApplyWorkersHash = hash_create("logical apply workers hash", + 16, &ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + } + + /* Try to get a free parallel apply worker. */ + foreach(lc, ParallelApplyWorkersList) + { + ParallelApplyWorkerInfo *tmp_winfo; + + tmp_winfo = (ParallelApplyWorkerInfo *) lfirst(lc); + + if (!tmp_winfo->in_use) + { + winfo = tmp_winfo; + break; + } + } + + if (winfo == NULL) + { + /* Try to start a new parallel apply worker. */ + winfo = parallel_apply_setup_worker(); + + if (winfo == NULL) + return; + } + + /* Create entry for requested transaction. */ + entry = hash_search(ParallelApplyWorkersHash, &xid, HASH_ENTER, &found); + if (found) + elog(ERROR, "hash table corrupted"); + + /* + * Set the in_parallel_apply_xact flag in the leader instead of the + * parallel apply worker to avoid the race condition where the leader has + * already started waiting for the parallel apply worker to finish + * processing the transaction while the child process has not yet + * processed the first STREAM_START and has not set the + * in_parallel_apply_xact to true. + */ + parallel_apply_set_in_xact(winfo->shared, true); + + winfo->in_use = true; + entry->winfo = winfo; + entry->xid = xid; +} + +/* + * Find the assigned worker for the given transaction, if any. + */ +ParallelApplyWorkerInfo * +parallel_apply_find_worker(TransactionId xid) +{ + bool found; + ParallelApplyWorkerEntry *entry; + + if (!TransactionIdIsValid(xid)) + return NULL; + + if (ParallelApplyWorkersHash == NULL) + return NULL; + + /* Return the cached parallel apply worker if valid. */ + if (stream_apply_worker != NULL) + return stream_apply_worker; + + /* + * Find entry for requested transaction. + */ + entry = hash_search(ParallelApplyWorkersHash, &xid, HASH_FIND, &found); + if (found) + { + Assert(parallel_apply_get_in_xact(entry->winfo->shared)); + Assert(entry->winfo->in_use); + + return entry->winfo; + } + + return NULL; +} + +/* + * Remove the parallel apply worker entry from the hash table. Stop the worker + * if there are enough workers in the pool. For more information about the + * worker pool, see comments atop worker.c. + */ +void +parallel_apply_free_worker(ParallelApplyWorkerInfo *winfo, TransactionId xid) +{ + int napplyworkers; + + Assert(!am_parallel_apply_worker()); + Assert(!parallel_apply_get_in_xact(winfo->shared)); + + if (!hash_search(ParallelApplyWorkersHash, &xid, HASH_REMOVE, NULL)) + elog(ERROR, "hash table corrupted"); + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + napplyworkers = logicalrep_parallel_apply_worker_count(MyLogicalRepWorker->subid); + LWLockRelease(LogicalRepWorkerLock); + + winfo->in_use = false; + + /* Are there enough workers in the pool? */ + if (napplyworkers > (max_parallel_apply_workers_per_subscription / 2)) + { + int slot_no; + uint16 generation; + + /* + * Detach the error queue before terminating the parallel apply worker + * to prevent the leader apply worker from receiving the worker + * termination message which will cause the leader to exit. + */ + shm_mq_detach(winfo->error_mq_handle); + winfo->error_mq_handle = NULL; + + SpinLockAcquire(&winfo->shared->mutex); + slot_no = winfo->shared->logicalrep_worker_slot_no; + generation = winfo->shared->logicalrep_worker_generation; + SpinLockRelease(&winfo->shared->mutex); + + logicalrep_worker_stop_by_slot(slot_no, generation); + + ParallelApplyWorkersList = list_delete_ptr(ParallelApplyWorkersList, + winfo); + + parallel_apply_free_worker_info(winfo); + } +} + +/* Free the parallel apply worker information. */ +static void +parallel_apply_free_worker_info(ParallelApplyWorkerInfo *winfo) +{ + Assert(winfo); + + if (winfo->mq_handle != NULL) + shm_mq_detach(winfo->mq_handle); + + if (winfo->error_mq_handle != NULL) + shm_mq_detach(winfo->error_mq_handle); + + if (winfo->dsm_seg != NULL) + dsm_detach(winfo->dsm_seg); + + pfree(winfo); +} + +/* Parallel apply worker main loop. */ +static void +LogicalParallelApplyLoop(shm_mq_handle *mqh) +{ + shm_mq_result shmq_res; + ErrorContextCallback errcallback; + MemoryContext oldcxt = CurrentMemoryContext; + + /* + * Init the ApplyMessageContext which we clean up after each replication + * protocol message. + */ + ApplyMessageContext = AllocSetContextCreate(ApplyContext, + "ApplyMessageContext", + ALLOCSET_DEFAULT_SIZES); + + /* + * Push apply error context callback. Fields will be filled while applying + * a change. + */ + errcallback.callback = apply_error_callback; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + for (;;) + { + void *data; + Size len; + + CHECK_FOR_INTERRUPTS(); + + /* Ensure we are reading the data into our memory context. */ + MemoryContextSwitchTo(ApplyMessageContext); + + shmq_res = shm_mq_receive(mqh, &len, &data, true); + + if (shmq_res == SHM_MQ_SUCCESS) + { + StringInfoData s; + int c; + + if (len == 0) + break; + + s.cursor = 0; + s.maxlen = -1; + s.data = (char *) data; + s.len = len; + + /* + * The first byte of messages sent from leader apply worker to + * parallel apply workers can only be 'w'. + */ + c = pq_getmsgbyte(&s); + if (c != 'w') + elog(ERROR, "unexpected message \"%c\"", c); + + /* + * Ignore statistics fields that have been updated by the leader + * apply worker. + */ + s.cursor += SIZE_STATS_MESSAGE; + + apply_dispatch(&s); + + MemoryContextReset(ApplyMessageContext); + } + else if (shmq_res == SHM_MQ_WOULD_BLOCK) + { + int rc; + + if (!in_streamed_transaction) + { + /* + * If we didn't get any transactions for a while there might be + * unconsumed invalidation messages in the queue, consume them + * now. + */ + AcceptInvalidationMessages(); + maybe_reread_subscription(); + } + + MemoryContextReset(ApplyMessageContext); + MemoryContextSwitchTo(oldcxt); + + /* Wait for more work. */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 1000L, + WAIT_EVENT_LOGICAL_PARALLEL_APPLY_MAIN); + + if (rc & WL_LATCH_SET) + { + ResetLatch(MyLatch); + CHECK_FOR_INTERRUPTS(); + } + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + } + } + else + { + Assert(shmq_res == SHM_MQ_DETACHED); + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("lost connection to the leader apply worker"))); + } + } + + /* Pop the error context stack. */ + error_context_stack = errcallback.previous; + + MemoryContextSwitchTo(oldcxt); +} + +/* + * Make sure the leader apply worker tries to read from our error queue one more + * time. This guards against the case where we exit uncleanly without sending + * an ErrorResponse, for example because some code calls proc_exit directly. + */ +static void +parallel_apply_shutdown(int code, Datum arg) +{ + SendProcSignal(MyLogicalRepWorker->apply_leader_pid, + PROCSIG_PARALLEL_APPLY_MESSAGE, + InvalidBackendId); + + dsm_detach((dsm_segment *) DatumGetPointer(arg)); +} + +/* + * Parallel apply worker entry point. + */ +void +ParallelApplyWorkerMain(Datum main_arg) +{ + ParallelApplyWorkerShared *shared; + dsm_handle handle; + dsm_segment *seg; + shm_toc *toc; + shm_mq *mq; + shm_mq_handle *mqh; + shm_mq_handle *error_mqh; + int worker_slot = DatumGetInt32(main_arg); + char originname[NAMEDATALEN]; + + /* Setup signal handling. */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + /* + * Attach to the dynamic shared memory segment for the parallel apply, and + * find its table of contents. + * + * Like parallel query, we don't need resource owner by this time. See + * ParallelWorkerMain. + */ + memcpy(&handle, MyBgworkerEntry->bgw_extra, sizeof(dsm_handle)); + seg = dsm_attach(handle); + if (seg == NULL) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("unable to map dynamic shared memory segment"))); + + toc = shm_toc_attach(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg)); + if (toc == NULL) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("bad magic number in dynamic shared memory segment"))); + + before_shmem_exit(parallel_apply_shutdown, PointerGetDatum(seg)); + + /* Look up the shared information. */ + shared = shm_toc_lookup(toc, PARALLEL_APPLY_KEY_SHARED, false); + MyParallelShared = shared; + + /* + * Attach to the message queue. + */ + mq = shm_toc_lookup(toc, PARALLEL_APPLY_KEY_MQ, false); + shm_mq_set_receiver(mq, MyProc); + mqh = shm_mq_attach(mq, seg, NULL); + + /* + * Primary initialization is complete. Now, we can attach to our slot. + * This is to ensure that the leader apply worker does not write data to + * the uninitialized memory queue. + */ + logicalrep_worker_attach(worker_slot); + + SpinLockAcquire(&MyParallelShared->mutex); + MyParallelShared->logicalrep_worker_generation = MyLogicalRepWorker->generation; + MyParallelShared->logicalrep_worker_slot_no = worker_slot; + SpinLockRelease(&MyParallelShared->mutex); + + /* + * Attach to the error queue. + */ + mq = shm_toc_lookup(toc, PARALLEL_APPLY_KEY_ERROR_QUEUE, false); + shm_mq_set_sender(mq, MyProc); + error_mqh = shm_mq_attach(mq, seg, NULL); + + pq_redirect_to_shm_mq(seg, error_mqh); + pq_set_parallel_leader(MyLogicalRepWorker->apply_leader_pid, + InvalidBackendId); + + MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time = + MyLogicalRepWorker->reply_time = 0; + + InitializeApplyWorker(); + + /* + * Setup callback for syscache so that we know when something changes in + * the subscription relation state. + */ + CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP, + invalidate_syncing_table_states, + (Datum) 0); + + /* + * Allocate the origin name in a long-lived context for error context + * message. + */ + ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid, + originname, sizeof(originname)); + apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext, + originname); + + LogicalParallelApplyLoop(mqh); + + proc_exit(0); +} + +/* + * Handle receipt of an interrupt indicating a parallel apply worker message. + * + * Note: this is called within a signal handler! All we can do is set a flag + * that will cause the next CHECK_FOR_INTERRUPTS() to invoke + * HandleParallelApplyMessages(). + */ +void +HandleParallelApplyMessageInterrupt(void) +{ + InterruptPending = true; + ParallelApplyMessagePending = true; + SetLatch(MyLatch); +} + +/* + * Handle a single protocol message received from a single parallel apply + * worker. + */ +static void +HandleParallelApplyMessage(ParallelApplyWorkerInfo *winfo, StringInfo msg) +{ + char msgtype; + + msgtype = pq_getmsgbyte(msg); + + switch (msgtype) + { + case 'E': /* ErrorResponse */ + { + ErrorData edata; + ErrorContextCallback *save_error_context_stack; + + /* Parse ErrorResponse. */ + pq_parse_errornotice(msg, &edata); + + /* Death of a worker isn't enough justification for suicide. */ + edata.elevel = Min(edata.elevel, ERROR); + + /* + * If desired, add a context line to show that this is a + * message propagated from a parallel apply worker. Otherwise, + * it can sometimes be confusing to understand what actually + * happened. + */ + if (edata.context) + edata.context = psprintf("%s\n%s", edata.context, + _("parallel apply worker")); + else + edata.context = pstrdup(_("parallel apply worker")); + + /* + * Context beyond that should use the error context callbacks + * that were in effect in LogicalRepApplyLoop(). + */ + save_error_context_stack = error_context_stack; + error_context_stack = apply_error_context_stack; + + ThrowErrorData(&edata); + + /* Should not reach here after rethrowing an error. */ + error_context_stack = save_error_context_stack; + + break; + } + + /* + * Don't need to do anything about NoticeResponse and + * NotifyResponse as the logical replication worker doesn't need + * to send messages to the client. + */ + case 'N': + case 'A': + break; + default: + elog(ERROR, "unrecognized message type received from parallel apply worker: %c (message length %d bytes)", + msgtype, msg->len); + } +} + +/* + * Handle any queued protocol messages received from parallel apply workers. + */ +void +HandleParallelApplyMessages(void) +{ + ListCell *lc; + MemoryContext oldcontext; + + static MemoryContext hpam_context = NULL; + + /* + * This is invoked from ProcessInterrupts(), and since some of the + * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential + * for recursive calls if more signals are received while this runs. It's + * unclear that recursive entry would be safe, and it doesn't seem useful + * even if it is safe, so let's block interrupts until done. + */ + HOLD_INTERRUPTS(); + + /* + * Moreover, CurrentMemoryContext might be pointing almost anywhere. We + * don't want to risk leaking data into long-lived contexts, so let's do + * our work here in a private context that we can reset on each use. + */ + if (hpam_context == NULL) /* first time through? */ + hpam_context = AllocSetContextCreate(TopMemoryContext, + "HandleParallelApplyMessages", + ALLOCSET_DEFAULT_SIZES); + else + MemoryContextReset(hpam_context); + + oldcontext = MemoryContextSwitchTo(hpam_context); + + ParallelApplyMessagePending = false; + + foreach(lc, ParallelApplyWorkersList) + { + shm_mq_result res; + Size nbytes; + void *data; + ParallelApplyWorkerInfo *winfo = (ParallelApplyWorkerInfo *) lfirst(lc); + + if (winfo->error_mq_handle == NULL) + continue; + + res = shm_mq_receive(winfo->error_mq_handle, &nbytes, &data, true); + + if (res == SHM_MQ_WOULD_BLOCK) + break; + else if (res == SHM_MQ_SUCCESS) + { + StringInfoData msg; + + initStringInfo(&msg); + appendBinaryStringInfo(&msg, data, nbytes); + HandleParallelApplyMessage(winfo, &msg); + pfree(msg.data); + } + else + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("lost connection to the parallel apply worker"))); + } + + MemoryContextSwitchTo(oldcontext); + + /* Might as well clear the context on our way out */ + MemoryContextReset(hpam_context); + + RESUME_INTERRUPTS(); +} + +/* + * Set up a dynamic shared memory segment. + * + * We set up a control region that contains a fixed-size worker info + * (ParallelApplyWorkerShared), a message queue, and an error queue. + * + * Returns true on success, false on failure. + */ +static bool +parallel_apply_setup_dsm(ParallelApplyWorkerInfo *winfo) +{ + shm_toc_estimator e; + Size segsize; + dsm_segment *seg; + shm_toc *toc; + ParallelApplyWorkerShared *shared; + shm_mq *mq; + Size queue_size = DSM_QUEUE_SIZE; + Size error_queue_size = DSM_ERROR_QUEUE_SIZE; + + /* + * Estimate how much shared memory we need. + * + * Because the TOC machinery may choose to insert padding of oddly-sized + * requests, we must estimate each chunk separately. + * + * We need one key to register the location of the header, and two other + * keys to track the locations of the message queue and the error message + * queue. + */ + shm_toc_initialize_estimator(&e); + shm_toc_estimate_chunk(&e, sizeof(ParallelApplyWorkerShared)); + shm_toc_estimate_chunk(&e, queue_size); + shm_toc_estimate_chunk(&e, error_queue_size); + + shm_toc_estimate_keys(&e, 3); + segsize = shm_toc_estimate(&e); + + /* Create the shared memory segment and establish a table of contents. */ + seg = dsm_create(shm_toc_estimate(&e), 0); + if (seg == NULL) + return false; + + toc = shm_toc_create(PG_LOGICAL_APPLY_SHM_MAGIC, dsm_segment_address(seg), + segsize); + + /* Set up the header region. */ + shared = shm_toc_allocate(toc, sizeof(ParallelApplyWorkerShared)); + SpinLockInit(&shared->mutex); + + shared->in_parallel_apply_xact = false; + + shm_toc_insert(toc, PARALLEL_APPLY_KEY_SHARED, shared); + + /* Set up message queue for the worker. */ + mq = shm_mq_create(shm_toc_allocate(toc, queue_size), queue_size); + shm_toc_insert(toc, PARALLEL_APPLY_KEY_MQ, mq); + shm_mq_set_sender(mq, MyProc); + + /* Attach the queue. */ + winfo->mq_handle = shm_mq_attach(mq, seg, NULL); + + /* Set up error queue for the worker. */ + mq = shm_mq_create(shm_toc_allocate(toc, error_queue_size), + error_queue_size); + shm_toc_insert(toc, PARALLEL_APPLY_KEY_ERROR_QUEUE, mq); + shm_mq_set_receiver(mq, MyProc); + + /* Attach the queue. */ + winfo->error_mq_handle = shm_mq_attach(mq, seg, NULL); + + /* Return results to caller. */ + winfo->dsm_seg = seg; + winfo->shared = shared; + + return true; +} + +/* + * Start parallel apply worker process and allocate shared memory for it. + */ +static ParallelApplyWorkerInfo * +parallel_apply_setup_worker(void) +{ + MemoryContext oldcontext; + bool launched; + ParallelApplyWorkerInfo *winfo; + + oldcontext = MemoryContextSwitchTo(ApplyContext); + + winfo = (ParallelApplyWorkerInfo *) palloc0(sizeof(ParallelApplyWorkerInfo)); + + /* Setup shared memory. */ + if (!parallel_apply_setup_dsm(winfo)) + { + MemoryContextSwitchTo(oldcontext); + pfree(winfo); + + return NULL; + } + + launched = logicalrep_worker_launch(MyLogicalRepWorker->dbid, + MySubscription->oid, + MySubscription->name, + MyLogicalRepWorker->userid, + InvalidOid, + dsm_segment_handle(winfo->dsm_seg)); + + if (launched) + { + ParallelApplyWorkersList = lappend(ParallelApplyWorkersList, winfo); + } + else + { + parallel_apply_free_worker_info(winfo); + + winfo = NULL; + } + + MemoryContextSwitchTo(oldcontext); + + return winfo; +} + +/* + * Send the data to the specified parallel apply worker via shared-memory queue. + */ +void +parallel_apply_send_data(ParallelApplyWorkerInfo *winfo, Size nbytes, + const void *data) +{ + shm_mq_result result; + + result = shm_mq_send(winfo->mq_handle, nbytes, data, false, true); + + if (result != SHM_MQ_SUCCESS) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("could not send data to shared-memory queue"))); +} + +/* + * Wait until the parallel apply worker processed the transaction finish command. + */ +void +parallel_apply_wait_for_xact_finish(ParallelApplyWorkerInfo *winfo) +{ + for (;;) + { + /* + * Stop if the parallel apply worker has processed the transaction + * finish command. + */ + if (!parallel_apply_get_in_xact(winfo->shared)) + break; + + /* Wait to be signalled. */ + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 1000L, + WAIT_EVENT_LOGICAL_PARALLEL_APPLY_STATE_CHANGE); + + /* Reset the latch so we don't spin. */ + ResetLatch(MyLatch); + + /* An interrupt may have occurred while we were waiting. */ + CHECK_FOR_INTERRUPTS(); + } +} + +/* + * Set the in_parallel_apply_xact flag for the given parallel apply worker. + */ +void +parallel_apply_set_in_xact(ParallelApplyWorkerShared *wshared, + bool in_xact) +{ + SpinLockAcquire(&wshared->mutex); + wshared->in_parallel_apply_xact = in_xact; + SpinLockRelease(&wshared->mutex); +} + +/* + * Get the in_parallel_apply_xact flag for the given parallel apply worker. + */ +static bool +parallel_apply_get_in_xact(ParallelApplyWorkerShared *wshared) +{ + bool in_xact; + + SpinLockAcquire(&wshared->mutex); + in_xact = wshared->in_parallel_apply_xact; + SpinLockRelease(&wshared->mutex); + + return in_xact; +} + +/* + * Form the savepoint name for the streaming transaction. + * + * Return the name in the supplied buffer. + */ +static void +parallel_apply_savepoint_name(Oid suboid, TransactionId xid, + char *spname, Size szsp) +{ + snprintf(spname, szsp, "pg_sp_%u_%u", suboid, xid); +} + +/* + * Define a savepoint for a subxact in parallel apply worker if needed. + * + * The parallel apply worker can figure out if a new subtransaction was + * started by checking if the new change arrived with a different xid. In that + * case define a named savepoint, so that we are able to rollback to it + * if required. + */ +void +parallel_apply_start_subtrans(TransactionId current_xid, TransactionId top_xid) +{ + if (current_xid != top_xid && + !list_member_xid(subxactlist, current_xid)) + { + MemoryContext oldctx; + char spname[NAMEDATALEN]; + + parallel_apply_savepoint_name(MySubscription->oid, current_xid, + spname, sizeof(spname)); + + elog(DEBUG1, "defining savepoint %s in parallel apply worker", spname); + + /* We must be in transaction block to define the SAVEPOINT. */ + if (!IsTransactionBlock()) + { + BeginTransactionBlock(); + CommitTransactionCommand(); + } + + DefineSavepoint(spname); + + /* + * CommitTransactionCommand is needed to start a subtransaction after + * issuing a SAVEPOINT inside a transaction block (see + * StartSubTransaction()). + */ + CommitTransactionCommand(); + + oldctx = MemoryContextSwitchTo(ApplyContext); + subxactlist = lappend_xid(subxactlist, current_xid); + MemoryContextSwitchTo(oldctx); + } +} + +/* + * Handle STREAM ABORT message when the transaction was applied in a parallel + * apply worker. + */ +void +parallel_apply_stream_abort(LogicalRepStreamAbortData *abort_data) +{ + TransactionId xid = abort_data->xid; + TransactionId subxid = abort_data->subxid; + + /* + * Update origin state so we can restart streaming from correct position + * in case of crash. + */ + replorigin_session_origin_lsn = abort_data->abort_lsn; + replorigin_session_origin_timestamp = abort_data->abort_time; + + /* + * If the two XIDs are the same, it's in fact abort of toplevel xact, so + * just free the subxactlist. + */ + if (subxid == xid) + { + parallel_apply_replorigin_setup(); + + AbortCurrentTransaction(); + + if (IsTransactionBlock()) + { + EndTransactionBlock(false); + CommitTransactionCommand(); + } + + parallel_apply_replorigin_reset(); + + pgstat_report_activity(STATE_IDLE, NULL); + + list_free(subxactlist); + subxactlist = NIL; + } + else + { + /* + * OK, so it's a subxact. Rollback to the savepoint. + * + * We also need to read the subxactlist, determine the offset tracked + * for the subxact, and truncate the list. + */ + int i; + bool found = false; + char spname[NAMEDATALEN]; + + parallel_apply_savepoint_name(MySubscription->oid, subxid, spname, + sizeof(spname)); + + elog(DEBUG1, "rolling back to savepoint %s in parallel apply worker", spname); + + for (i = list_length(subxactlist) - 1; i >= 0; i--) + { + TransactionId xid_tmp = lfirst_xid(list_nth_cell(subxactlist, i)); + + if (xid_tmp == subxid) + { + found = true; + break; + } + } + + if (found) + { + RollbackToSavepoint(spname); + CommitTransactionCommand(); + subxactlist = list_truncate(subxactlist, i + 1); + } + + pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL); + } + + parallel_apply_set_in_xact(MyParallelShared, false); +} + +/* Setup replication origin tracking. */ +void +parallel_apply_replorigin_setup(void) +{ + RepOriginId originid; + char originname[NAMEDATALEN]; + bool started_tx = false; + + /* This function might be called inside or outside of transaction. */ + if (!IsTransactionState()) + { + StartTransactionCommand(); + started_tx = true; + } + + ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid, + originname, sizeof(originname)); + originid = replorigin_by_name(originname, false); + replorigin_session_setup(originid); + replorigin_session_origin = originid; + + if (started_tx) + CommitTransactionCommand(); +} + +/* Reset replication origin tracking. */ +void +parallel_apply_replorigin_reset(void) +{ + replorigin_session_reset(); + + replorigin_session_origin = InvalidRepOriginId; + replorigin_session_origin_lsn = InvalidXLogRecPtr; + replorigin_session_origin_timestamp = 0; +} diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c index 2cc0ac9..303557d 100644 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@ -652,9 +652,10 @@ DecodeCommit(LogicalDecodingContext *ctx, XLogRecordBuffer *buf, { for (i = 0; i < parsed->nsubxacts; i++) { - ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr); + ReorderBufferForget(ctx->reorder, parsed->subxacts[i], buf->origptr, + commit_time); } - ReorderBufferForget(ctx->reorder, xid, buf->origptr); + ReorderBufferForget(ctx->reorder, xid, buf->origptr, commit_time); return; } @@ -822,10 +823,11 @@ DecodeAbort(LogicalDecodingContext *ctx, XLogRecordBuffer *buf, for (i = 0; i < parsed->nsubxacts; i++) { ReorderBufferAbort(ctx->reorder, parsed->subxacts[i], - buf->record->EndRecPtr); + buf->record->EndRecPtr, abort_time); } - ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr); + ReorderBufferAbort(ctx->reorder, xid, buf->record->EndRecPtr, + abort_time); } /* update the decoding stats */ diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index f2c55f3..7cbd2f6 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -54,6 +54,7 @@ int max_logical_replication_workers = 4; int max_sync_workers_per_subscription = 2; +int max_parallel_apply_workers_per_subscription = 2; LogicalRepWorker *MyLogicalRepWorker = NULL; @@ -151,8 +152,10 @@ get_subscription_list(void) * * This is only needed for cleaning up the shared memory in case the worker * fails to attach. + * + * Return whether the attach was successful. */ -static void +static bool WaitForReplicationWorkerAttach(LogicalRepWorker *worker, uint16 generation, BackgroundWorkerHandle *handle) @@ -168,11 +171,11 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - /* Worker either died or has started; no need to do anything. */ + /* Worker either died or has started. Return false if died. */ if (!worker->in_use || worker->proc) { LWLockRelease(LogicalRepWorkerLock); - return; + return worker->in_use; } LWLockRelease(LogicalRepWorkerLock); @@ -187,7 +190,7 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, if (generation == worker->generation) logicalrep_worker_cleanup(worker); LWLockRelease(LogicalRepWorkerLock); - return; + return false; } /* @@ -209,6 +212,8 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, /* * Walks the workers array and searches for one that matches given * subscription id and relid. + * + * We are only interested in the leader apply worker or table sync worker. */ LogicalRepWorker * logicalrep_worker_find(Oid subid, Oid relid, bool only_running) @@ -223,6 +228,10 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; + /* Skip parallel apply workers. */ + if (isParallelApplyWorker(w)) + continue; + if (w->in_use && w->subid == subid && w->relid == relid && (!only_running || w->proc)) { @@ -259,11 +268,13 @@ logicalrep_workers_find(Oid subid, bool only_running) } /* - * Start new apply background worker, if possible. + * Start new background worker, if possible. + * + * Returns true on success, false on failure. */ -void +bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid, - Oid relid) + Oid relid, dsm_handle subworker_dsm) { BackgroundWorker bgw; BackgroundWorkerHandle *bgw_handle; @@ -272,7 +283,12 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid, int slot = 0; LogicalRepWorker *worker = NULL; int nsyncworkers; + int nparallelapplyworkers; TimestampTz now; + bool is_parallel_apply_worker = (subworker_dsm != DSM_HANDLE_INVALID); + + /* Sanity check - tablesync worker cannot be a subworker */ + Assert(!(is_parallel_apply_worker && OidIsValid(relid))); ereport(DEBUG1, (errmsg_internal("starting logical replication worker for subscription \"%s\"", @@ -350,7 +366,26 @@ retry: if (OidIsValid(relid) && nsyncworkers >= max_sync_workers_per_subscription) { LWLockRelease(LogicalRepWorkerLock); - return; + return false; + } + + nparallelapplyworkers = logicalrep_parallel_apply_worker_count(subid); + + /* + * Return false if the number of parallel apply workers reached the limit + * per subscription. + */ + if (is_parallel_apply_worker && + nparallelapplyworkers >= max_parallel_apply_workers_per_subscription) + { + LWLockRelease(LogicalRepWorkerLock); + + ereport(LOG, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of parallel apply workers"), + errhint("You might need to increase max_parallel_apply_workers_per_subscription."))); + + return false; } /* @@ -364,7 +399,7 @@ retry: (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("out of logical replication worker slots"), errhint("You might need to increase max_logical_replication_workers."))); - return; + return false; } /* Prepare the worker slot. */ @@ -379,6 +414,8 @@ retry: worker->relstate = SUBREL_STATE_UNKNOWN; worker->relstate_lsn = InvalidXLogRecPtr; worker->stream_fileset = NULL; + worker->apply_leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid; + worker->parallel_apply = is_parallel_apply_worker; worker->last_lsn = InvalidXLogRecPtr; TIMESTAMP_NOBEGIN(worker->last_send_time); TIMESTAMP_NOBEGIN(worker->last_recv_time); @@ -396,19 +433,34 @@ retry: BGWORKER_BACKEND_DATABASE_CONNECTION; bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain"); + + if (is_parallel_apply_worker) + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain"); + else + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain"); + if (OidIsValid(relid)) snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication worker for subscription %u sync %u", subid, relid); + else if (is_parallel_apply_worker) + snprintf(bgw.bgw_name, BGW_MAXLEN, + "logical replication parallel apply worker for subscription %u", subid); else snprintf(bgw.bgw_name, BGW_MAXLEN, - "logical replication worker for subscription %u", subid); - snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker"); + "logical replication apply worker for subscription %u", subid); + + if (is_parallel_apply_worker) + snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker"); + else + snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker"); bgw.bgw_restart_time = BGW_NEVER_RESTART; bgw.bgw_notify_pid = MyProcPid; bgw.bgw_main_arg = Int32GetDatum(slot); + if (is_parallel_apply_worker) + memcpy(bgw.bgw_extra, &subworker_dsm, sizeof(dsm_handle)); + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) { /* Failed to start worker, so clean up the worker slot. */ @@ -421,33 +473,22 @@ retry: (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("out of background worker slots"), errhint("You might need to increase max_worker_processes."))); - return; + return false; } /* Now wait until it attaches. */ - WaitForReplicationWorkerAttach(worker, generation, bgw_handle); + return WaitForReplicationWorkerAttach(worker, generation, bgw_handle); } /* - * Stop the logical replication worker for subid/relid, if any, and wait until - * it detaches from the slot. + * Internal function to stop the worker and wait for it to die. */ -void -logicalrep_worker_stop(Oid subid, Oid relid) +static void +logicalrep_worker_stop_internal(LogicalRepWorker *worker) { - LogicalRepWorker *worker; uint16 generation; - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - - worker = logicalrep_worker_find(subid, relid, false); - - /* No worker, nothing to do. */ - if (!worker) - { - LWLockRelease(LogicalRepWorkerLock); - return; - } + Assert(LWLockHeldByMeInMode(LogicalRepWorkerLock, LW_SHARED)); /* * Remember which generation was our worker so we can check if what we see @@ -485,10 +526,7 @@ logicalrep_worker_stop(Oid subid, Oid relid) * different, meaning that a different worker has taken the slot. */ if (!worker->in_use || worker->generation != generation) - { - LWLockRelease(LogicalRepWorkerLock); return; - } /* Worker has assigned proc, so it has started. */ if (worker->proc) @@ -522,6 +560,50 @@ logicalrep_worker_stop(Oid subid, Oid relid) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); } +} + +/* + * Stop the logical replication worker for subid/relid, if any, and wait until + * it detaches from the slot. + */ +void +logicalrep_worker_stop(Oid subid, Oid relid) +{ + LogicalRepWorker *worker; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + + worker = logicalrep_worker_find(subid, relid, false); + + if (worker) + { + Assert(!isParallelApplyWorker(worker)); + logicalrep_worker_stop_internal(worker); + } + + LWLockRelease(LogicalRepWorkerLock); +} + +/* + * Stop the logical replication worker corresponding to the input slot number, + * and wait until it detaches from the slot. + */ +void +logicalrep_worker_stop_by_slot(int slot_no, uint16 generation) +{ + LogicalRepWorker *worker; + + Assert(slot_no >=0 && slot_no < max_logical_replication_workers); + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + + worker = &LogicalRepCtx->workers[slot_no]; + + /* + * Only stop the worker if the generation matches and the worker is alive. + */ + if (worker->generation == generation && worker->proc) + logicalrep_worker_stop_internal(worker); LWLockRelease(LogicalRepWorkerLock); } @@ -594,11 +676,32 @@ logicalrep_worker_attach(int slot) } /* - * Detach the worker (cleans up the worker info). + * Stop the parallel apply workers if any, and detach the leader apply worker + * (cleans up the worker info). */ static void logicalrep_worker_detach(void) { + /* Stop the parallel apply workers. */ + if (am_leader_apply_worker()) + { + List *workers; + ListCell *lc; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + + workers = logicalrep_workers_find(MyLogicalRepWorker->subid, true); + foreach(lc, workers) + { + LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); + + if (isParallelApplyWorker(w)) + logicalrep_worker_stop_internal(w); + } + + LWLockRelease(LogicalRepWorkerLock); + } + /* Block concurrent access. */ LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE); @@ -621,6 +724,8 @@ logicalrep_worker_cleanup(LogicalRepWorker *worker) worker->userid = InvalidOid; worker->subid = InvalidOid; worker->relid = InvalidOid; + worker->apply_leader_pid = InvalidPid; + worker->parallel_apply = false; } /* @@ -680,6 +785,33 @@ logicalrep_sync_worker_count(Oid subid) } /* + * Count the number of registered (but not necessarily running) parallel apply + * workers for a subscription. + */ +int +logicalrep_parallel_apply_worker_count(Oid subid) +{ + int i; + int res = 0; + + Assert(LWLockHeldByMe(LogicalRepWorkerLock)); + + /* + * Scan all attached parallel apply workers, only counting those which + * have the given subscription id. + */ + for (i = 0; i < max_logical_replication_workers; i++) + { + LogicalRepWorker *w = &LogicalRepCtx->workers[i]; + + if (w->subid == subid && isParallelApplyWorker(w)) + res++; + } + + return res; +} + +/* * ApplyLauncherShmemSize * Compute space needed for replication launcher shared memory */ @@ -868,7 +1000,7 @@ ApplyLauncherMain(Datum main_arg) wait_time = wal_retrieve_retry_interval; logicalrep_worker_launch(sub->dbid, sub->oid, sub->name, - sub->owner, InvalidOid); + sub->owner, InvalidOid, DSM_HANDLE_INVALID); } } @@ -951,6 +1083,10 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) if (OidIsValid(subid) && worker.subid != subid) continue; + /* Skip if this is a parallel apply worker */ + if (isParallelApplyWorker(&worker)) + continue; + worker_pid = worker.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index 773583a..26bfc0e 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -1,4 +1,5 @@ backend_sources += files( + 'applyparallelworker.c', 'decode.c', 'launcher.c', 'logical.c', diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index ff8513e..d6160e6 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -1163,10 +1163,14 @@ logicalrep_read_stream_commit(StringInfo in, LogicalRepCommitData *commit_data) /* * Write STREAM ABORT to the output stream. Note that xid and subxid will be * same for the top-level transaction abort. + * + * If write_abort_info is true, send the abort_lsn and abort_time fields, + * otherwise don't. */ void logicalrep_write_stream_abort(StringInfo out, TransactionId xid, - TransactionId subxid) + TransactionId subxid, XLogRecPtr abort_lsn, + TimestampTz abort_time, bool write_abort_info) { pq_sendbyte(out, LOGICAL_REP_MSG_STREAM_ABORT); @@ -1175,19 +1179,40 @@ logicalrep_write_stream_abort(StringInfo out, TransactionId xid, /* transaction ID */ pq_sendint32(out, xid); pq_sendint32(out, subxid); + + if (write_abort_info) + { + pq_sendint64(out, abort_lsn); + pq_sendint64(out, abort_time); + } } /* * Read STREAM ABORT from the output stream. + * + * If read_abort_info is true, read the abort_lsn and abort_time fields, + * otherwise don't. */ void -logicalrep_read_stream_abort(StringInfo in, TransactionId *xid, - TransactionId *subxid) +logicalrep_read_stream_abort(StringInfo in, + LogicalRepStreamAbortData *abort_data, + bool read_abort_info) { - Assert(xid && subxid); + Assert(abort_data); + + abort_data->xid = pq_getmsgint(in, 4); + abort_data->subxid = pq_getmsgint(in, 4); - *xid = pq_getmsgint(in, 4); - *subxid = pq_getmsgint(in, 4); + if (read_abort_info) + { + abort_data->abort_lsn = pq_getmsgint64(in); + abort_data->abort_time = pq_getmsgint64(in); + } + else + { + abort_data->abort_lsn = InvalidXLogRecPtr; + abort_data->abort_time = 0; + } } /* diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index f55bf44..30e8fe5 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -2853,7 +2853,8 @@ ReorderBufferFinishPrepared(ReorderBuffer *rb, TransactionId xid, * disk. */ void -ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn) +ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn, + TimestampTz abort_time) { ReorderBufferTXN *txn; @@ -2864,6 +2865,8 @@ ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn) if (txn == NULL) return; + txn->xact_time.abort_time = abort_time; + /* For streamed transactions notify the remote node about the abort. */ if (rbtxn_is_streamed(txn)) { @@ -2938,7 +2941,8 @@ ReorderBufferAbortOld(ReorderBuffer *rb, TransactionId oldestRunningXid) * to this xid might re-create the transaction incompletely. */ void -ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn) +ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn, + TimestampTz abort_time) { ReorderBufferTXN *txn; @@ -2949,6 +2953,8 @@ ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn) if (txn == NULL) return; + txn->xact_time.abort_time = abort_time; + /* For streamed transactions notify the remote node about the abort. */ if (rbtxn_is_streamed(txn)) rb->stream_abort(rb, txn, lsn); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 94e813a..1c1a8e7 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -14,7 +14,7 @@ * The initial data synchronization is done separately for each table, * in a separate apply worker that only fetches the initial snapshot data * from the publisher and then synchronizes the position in the stream with - * the main apply worker. + * the leader apply worker. * * There are several reasons for doing the synchronization this way: * - It allows us to parallelize the initial data synchronization @@ -153,7 +153,7 @@ finish_sync_worker(void) get_rel_name(MyLogicalRepWorker->relid)))); CommitTransactionCommand(); - /* Find the main apply worker and signal it. */ + /* Find the leader apply worker and signal it. */ logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid); /* Stop gracefully */ @@ -609,7 +609,8 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) MySubscription->oid, MySubscription->name, MyLogicalRepWorker->userid, - rstate->relid); + rstate->relid, + DSM_HANDLE_INVALID); hentry->last_start_time = now; } } @@ -630,6 +631,13 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) void process_syncing_tables(XLogRecPtr current_lsn) { + /* + * Skip for parallel apply workers. See parallel_apply_can_start() for + * details. + */ + if (am_parallel_apply_worker()) + return; + if (am_tablesync_worker()) process_syncing_tables_for_sync(current_lsn); else @@ -1247,7 +1255,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) /* * Here we use the slot name instead of the subscription name as the - * application_name, so that it is different from the main apply worker, + * application_name, so that it is different from the leader apply worker, * so that synchronous replication can distinguish them. */ LogRepWorkerWalRcvConn = @@ -1461,8 +1469,8 @@ copy_table_done: SpinLockRelease(&MyLogicalRepWorker->relmutex); /* - * Finally, wait until the main apply worker tells us to catch up and then - * return to let LogicalRepApplyLoop do it. + * Finally, wait until the leader apply worker tells us to catch up and + * then return to let LogicalRepApplyLoop do it. */ wait_for_worker_state_change(SUBREL_STATE_CATCHUP); return slotname; diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 5250ae7..f131ace 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -22,8 +22,55 @@ * STREAMED TRANSACTIONS * --------------------- * Streamed transactions (large transactions exceeding a memory limit on the - * upstream) are not applied immediately, but instead, the data is written - * to temporary files and then applied at once when the final commit arrives. + * upstream) are applied using one of two approaches: + * + * 1) Parallel apply workers + * + * If streaming = parallel, we assign a new parallel apply worker (if + * available) as soon as the xact's first stream is received. The leader apply + * worker will send changes to this new worker via shared memory. We keep this + * worker assigned till the transaction commit is received and also wait for + * the worker to finish at commit. This preserves commit ordering and avoids + * file I/O in most cases, although we still need to spill to a file if there + * is no worker available. It is important to maintain commit order to avoid + * failures due to (a) transaction dependencies, say if we insert a row in the + * first transaction and update it in the second transaction then allowing to + * apply both in parallel can lead to failure in the update. (b) deadlocks, + * allowing transactions that update the same set of rows/tables in opposite + * order to be applied in parallel can lead to deadlocks. + * + * We maintain a worker pool to avoid restarting workers for each streaming + * transaction. We maintain each worker's information in the + * ParallelApplyWorkersList. After successfully launching a new worker, its + * information is added to the ParallelApplyWorkersList. Once the worker + * finishes applying the transaction, we mark it available for re-use. Now, + * before starting a new worker to apply the streaming transaction, we check + * the list for any available worker. Note that we maintain a maximum of half + * the max_parallel_apply_workers_per_subscription workers in the pool and + * after that, we simply exit the worker after applying the transaction. + * + * XXX This worker pool threshold is a bit arbitrary and we can provide a GUC + * variable for this in the future if required. + * + * The leader apply worker will create separate dynamic shared memory segment + * when each parallel apply worker starts. The reason for this design is that + * we cannot count how many workers will be started. It may be possible to + * allocate enough shared memory in one segment based on the maximum number of + * parallel apply workers (max_parallel_apply_workers_per_subscription), but this + * would waste memory if no process is actually started. + * + * The dynamic shared memory segment will contain (a) a shm_mq that can be + * used to send changes in the transaction from leader apply worker to parallel + * apply worker (b) another shm_mq that can be used to send errors (and other + * messages reported via elog/ereport) from the parallel apply worker to leader + * apply worker (c) necessary information to be shared among parallel apply + * workers and leader apply worker (i.e. in_parallel_apply_xact flag and the + * corresponding LogicalRepWorker slot information). + * + * If no parallel apply worker is available to handle the streamed transaction + * we follow approach 2. + * + * 2) Write to temporary files and apply when the final commit arrives * * Unlike the regular (non-streamed) case, handling streamed transactions has * to handle aborts of both the toplevel transaction and subtransactions. This @@ -219,20 +266,35 @@ typedef struct ApplyExecutionData PartitionTupleRouting *proute; /* partition routing info */ } ApplyExecutionData; -/* Struct for saving and restoring apply errcontext information */ -typedef struct ApplyErrorCallbackArg +/* + * What action to take for the transaction. + * + * TRANS_LEADER_APPLY means that we are in the leader apply worker and changes + * of the transaction are applied directly in the worker. + * + * TRANS_LEADER_SERIALIZE means that we are in the leader apply worker or table + * sync worker. Changes are written to temporary files and then applied when + * the final commit arrives. + * + * TRANS_LEADER_SEND_TO_PARALLEL means that we are in the leader apply worker + * and need to send the changes to the parallel apply worker. + * + * TRANS_PARALLEL_APPLY means that we are in the parallel apply worker and + * changes of the transaction are applied directly in the worker. + */ +typedef enum { - LogicalRepMsgType command; /* 0 if invalid */ - LogicalRepRelMapEntry *rel; + /* The action for non-streaming transactions. */ + TRANS_LEADER_APPLY, - /* Remote node information */ - int remote_attnum; /* -1 if invalid */ - TransactionId remote_xid; - XLogRecPtr finish_lsn; - char *origin_name; -} ApplyErrorCallbackArg; + /* Actions for streaming transactions. */ + TRANS_LEADER_SERIALIZE, + TRANS_LEADER_SEND_TO_PARALLEL, + TRANS_PARALLEL_APPLY +} TransApplyAction; -static ApplyErrorCallbackArg apply_error_callback_arg = +/* errcontext tracker */ +ApplyErrorCallbackArg apply_error_callback_arg = { .command = 0, .rel = NULL, @@ -242,7 +304,9 @@ static ApplyErrorCallbackArg apply_error_callback_arg = .origin_name = NULL, }; -static MemoryContext ApplyMessageContext = NULL; +ErrorContextCallback *apply_error_context_stack = NULL; + +MemoryContext ApplyMessageContext = NULL; MemoryContext ApplyContext = NULL; /* per stream context for streaming transactions */ @@ -251,27 +315,35 @@ static MemoryContext LogicalStreamingContext = NULL; WalReceiverConn *LogRepWorkerWalRcvConn = NULL; Subscription *MySubscription = NULL; -static bool MySubscriptionValid = false; +bool MySubscriptionValid = false; bool in_remote_transaction = false; static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr; /* fields valid only when processing streamed transaction */ -static bool in_streamed_transaction = false; +bool in_streamed_transaction = false; static TransactionId stream_xid = InvalidTransactionId; /* + * The number of changes sent to parallel apply workers during one streaming + * block. + */ +static uint32 parallel_stream_nchanges = 0; + +/* * We enable skipping all data modification changes (INSERT, UPDATE, etc.) for * the subscription if the remote transaction's finish LSN matches the subskiplsn. * Once we start skipping changes, we don't stop it until we skip all changes of * the transaction even if pg_subscription is updated and MySubscription->skiplsn - * gets changed or reset during that. Also, in streaming transaction cases, we - * don't skip receiving and spooling the changes since we decide whether or not + * gets changed or reset during that. Also, in streaming transaction cases (streaming = on), + * we don't skip receiving and spooling the changes since we decide whether or not * to skip applying the changes when starting to apply changes. The subskiplsn is * cleared after successfully skipping the transaction or applying non-empty * transaction. The latter prevents the mistakenly specified subskiplsn from - * being left. + * being left. Note that we cannot skip the streaming transactions when using + * parallel apply workers because we cannot get the finish LSN before + * applying the changes. */ static XLogRecPtr skip_xact_finish_lsn = InvalidXLogRecPtr; #define is_skipping_changes() (unlikely(!XLogRecPtrIsInvalid(skip_xact_finish_lsn))) @@ -321,13 +393,8 @@ static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply); static void store_flush_position(XLogRecPtr remote_lsn); -static void maybe_reread_subscription(void); - static void DisableSubscriptionAndExit(void); -/* prototype needed because of stream_commit */ -static void apply_dispatch(StringInfo s); - static void apply_handle_commit_internal(LogicalRepCommitData *commit_data); static void apply_handle_insert_internal(ApplyExecutionData *edata, ResultRelInfo *relinfo, @@ -360,10 +427,12 @@ static void stop_skipping_changes(void); static void clear_subscription_skip_lsn(XLogRecPtr finish_lsn); /* Functions for apply error callback */ -static void apply_error_callback(void *arg); static inline void set_apply_error_context_xact(TransactionId xid, XLogRecPtr lsn); static inline void reset_apply_error_context_info(void); +static TransApplyAction get_transaction_apply_action(TransactionId xid, + ParallelApplyWorkerInfo **winfo); + /* * Form the origin name for the subscription. * @@ -393,19 +462,42 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, * * This is mainly needed for initial relation data sync as that runs in * separate worker process running in parallel and we need some way to skip - * changes coming to the main apply worker during the sync of a table. + * changes coming to the leader apply worker during the sync of a table. * * Note we need to do smaller or equals comparison for SYNCDONE state because * it might hold position of end of initial slot consistent point WAL * record + 1 (ie start of next record) and next record can be COMMIT of * transaction we are now processing (which is what we set remote_final_lsn * to in apply_handle_begin). + * + * Note that for streaming transactions that are being applied in the parallel + * apply worker, we disallow applying changes on a table that is not in + * the READY state, because we cannot decide whether to apply the change as we + * won't know remote_final_lsn by that time. + * + * We already checked this in parallel_apply_can_start() before assigning the + * streaming transaction to the parallel worker, but it also needs to be + * checked here because if the user executes ALTER SUBSCRIPTION ... REFRESH + * PUBLICATION in parallel, the new table can be added to pg_subscription_rel + * while applying this transaction. */ static bool should_apply_changes_for_rel(LogicalRepRelMapEntry *rel) { if (am_tablesync_worker()) return MyLogicalRepWorker->relid == rel->localreloid; + else if (am_parallel_apply_worker()) + { + if (rel->state != SUBREL_STATE_READY) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("logical replication parallel apply worker for subscription \"%s\" will stop", + MySubscription->name), + errdetail("Cannot handle streamed replication transaction using parallel " + "apply workers until all tables are synchronized."))); + + return true; + } else return (rel->state == SUBREL_STATE_READY || (rel->state == SUBREL_STATE_SYNCDONE && @@ -451,43 +543,85 @@ end_replication_step(void) } /* - * Handle streamed transactions. + * Handle streamed transactions for both the leader apply worker and the parallel + * apply workers. + * + * In streaming case (receiving a block of streamed transaction), for + * SUBSTREAM_ON mode, simply redirect it to a file for the proper toplevel + * transaction, and for SUBSTREAM_PARALLEL mode, send the changes to parallel + * apply workers (LOGICAL_REP_MSG_RELATION or LOGICAL_REP_MSG_TYPE changes + * will be applied by both leader apply worker and parallel apply workers). * - * If in streaming mode (receiving a block of streamed transaction), we - * simply redirect it to a file for the proper toplevel transaction. + * For non-streamed transactions, returns false. + * For streamed transactions, returns true if in leader apply worker, false + * otherwise. * - * Returns true for streamed transactions, false otherwise (regular mode). + * Exception: If the message being processed is LOGICAL_REP_MSG_RELATION + * or LOGICAL_REP_MSG_TYPE, return false even if the message needs to be sent + * to a parallel apply worker. */ static bool handle_streamed_transaction(LogicalRepMsgType action, StringInfo s) { - TransactionId xid; + TransactionId current_xid; + ParallelApplyWorkerInfo *winfo; + TransApplyAction apply_action; + + apply_action = get_transaction_apply_action(stream_xid, &winfo); /* not in streaming mode */ - if (!in_streamed_transaction) + if (apply_action == TRANS_LEADER_APPLY) return false; - Assert(stream_fd != NULL); Assert(TransactionIdIsValid(stream_xid)); /* * We should have received XID of the subxact as the first part of the * message, so extract it. */ - xid = pq_getmsgint(s, 4); + current_xid = pq_getmsgint(s, 4); - if (!TransactionIdIsValid(xid)) + if (!TransactionIdIsValid(current_xid)) ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg_internal("invalid transaction ID in streamed replication transaction"))); - /* Add the new subxact to the array (unless already there). */ - subxact_info_add(xid); + switch (apply_action) + { + case TRANS_LEADER_SERIALIZE: + Assert(stream_fd != NULL); + + /* Add the new subxact to the array (unless already there). */ + subxact_info_add(current_xid); + + /* Write the change to the current file */ + stream_write_change(action, s); + return true; + + case TRANS_LEADER_SEND_TO_PARALLEL: + Assert(winfo); + + parallel_apply_send_data(winfo, s->len, s->data); + parallel_stream_nchanges += 1; + + /* + * XXX The publisher side doesn't always send relation/type update + * messages after the streaming transaction, so also update the + * relation/type in leader apply worker. See function + * cleanup_rel_sync_cache. + */ + return (action != LOGICAL_REP_MSG_RELATION && + action != LOGICAL_REP_MSG_TYPE); - /* write the change to the current file */ - stream_write_change(action, s); + case TRANS_PARALLEL_APPLY: + /* Define a savepoint for a subxact if needed. */ + parallel_apply_start_subtrans(current_xid, stream_xid); + return false; - return true; + default: + Assert(false); + return false; /* silence compiler warning */ + } } /* @@ -923,8 +1057,11 @@ apply_handle_prepare_internal(LogicalRepPreparedTxnData *prepare_data) * BeginTransactionBlock is necessary to balance the EndTransactionBlock * called within the PrepareTransactionBlock below. */ - BeginTransactionBlock(); - CommitTransactionCommand(); /* Completes the preceding Begin command. */ + if (!IsTransactionBlock()) + { + BeginTransactionBlock(); + CommitTransactionCommand(); /* Completes the preceding Begin command. */ + } /* * Update origin state so we can restart streaming from correct position @@ -993,6 +1130,12 @@ apply_handle_prepare(StringInfo s) /* * Handle a COMMIT PREPARED of a previously PREPARED transaction. + * + * Note that we don't need to wait here if the transaction was prepared in a + * parallel apply worker. In that case, we have already waited for the prepare + * to finish in apply_handle_stream_prepare() which will ensure all the + * operations in that transaction have happened in the subscriber, so no + * concurrent transaction can cause deadlock or transaction dependency issues. */ static void apply_handle_commit_prepared(StringInfo s) @@ -1036,6 +1179,12 @@ apply_handle_commit_prepared(StringInfo s) /* * Handle a ROLLBACK PREPARED of a previously PREPARED TRANSACTION. + * + * Note that we don't need to wait here if the transaction was prepared in a + * parallel apply worker. In that case, we have already waited for the prepare + * to finish in apply_handle_stream_prepare() which will ensure all the + * operations in that transaction have happened in the subscriber, so no + * concurrent transaction can cause deadlock or transaction dependency issues. */ static void apply_handle_rollback_prepared(StringInfo s) @@ -1089,15 +1238,13 @@ apply_handle_rollback_prepared(StringInfo s) /* * Handle STREAM PREPARE. - * - * Logic is in two parts: - * 1. Replay all the spooled operations - * 2. Mark the transaction as prepared */ static void apply_handle_stream_prepare(StringInfo s) { LogicalRepPreparedTxnData prepare_data; + ParallelApplyWorkerInfo *winfo; + TransApplyAction apply_action; if (in_streamed_transaction) ereport(ERROR, @@ -1113,24 +1260,82 @@ apply_handle_stream_prepare(StringInfo s) logicalrep_read_stream_prepare(s, &prepare_data); set_apply_error_context_xact(prepare_data.xid, prepare_data.prepare_lsn); - elog(DEBUG1, "received prepare for streamed transaction %u", prepare_data.xid); + apply_action = get_transaction_apply_action(prepare_data.xid, &winfo); - /* Replay all the spooled operations. */ - apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn); + switch (apply_action) + { + case TRANS_LEADER_SERIALIZE: - /* Mark the transaction as prepared. */ - apply_handle_prepare_internal(&prepare_data); + /* + * The transaction has been serialized to file, so replay all the + * spooled operations. + */ + apply_spooled_messages(prepare_data.xid, prepare_data.prepare_lsn); - CommitTransactionCommand(); + /* Mark the transaction as prepared. */ + apply_handle_prepare_internal(&prepare_data); - pgstat_report_stat(false); + CommitTransactionCommand(); - store_flush_position(prepare_data.end_lsn); + store_flush_position(prepare_data.end_lsn); - in_remote_transaction = false; + in_remote_transaction = false; + + /* Unlink the files with serialized changes and subxact info. */ + stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid); + break; + + case TRANS_LEADER_SEND_TO_PARALLEL: + Assert(winfo); + + /* + * The origin can be active only in one process. See + * apply_handle_stream_commit. + */ + parallel_apply_replorigin_reset(); + + /* Send STREAM PREPARE message to the parallel apply worker. */ + parallel_apply_send_data(winfo, s->len, s->data); + + /* + * After sending the data to the parallel apply worker, wait for + * that worker to finish. This is necessary to maintain commit + * order which avoids failures due to transaction dependencies and + * deadlocks. + */ + parallel_apply_wait_for_xact_finish(winfo); + parallel_apply_replorigin_setup(); + parallel_apply_free_worker(winfo, prepare_data.xid); + + in_remote_transaction = false; + + store_flush_position(prepare_data.end_lsn); + break; - /* unlink the files with serialized changes and subxact info. */ - stream_cleanup_files(MyLogicalRepWorker->subid, prepare_data.xid); + case TRANS_PARALLEL_APPLY: + parallel_apply_replorigin_setup(); + + /* Mark the transaction as prepared. */ + apply_handle_prepare_internal(&prepare_data); + + CommitTransactionCommand(); + + parallel_apply_replorigin_reset(); + + list_free(subxactlist); + subxactlist = NIL; + + parallel_apply_set_in_xact(MyParallelShared, false); + + elog(DEBUG1, "finished processing the transaction finish command"); + break; + + default: + Assert(false); + break; + } + + pgstat_report_stat(false); /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(prepare_data.end_lsn); @@ -1169,26 +1374,25 @@ apply_handle_origin(StringInfo s) /* * Handle STREAM START message. + * + * XXX We can avoid sending pairs of the START/STOP messages to the parallel + * worker because unlike apply worker it will process only one transaction at a + * time. However, it is not clear whether any optimization is worthwhile + * because these messages are sent only when the logical_decoding_work_mem + * threshold is exceeded. */ static void apply_handle_stream_start(StringInfo s) { bool first_segment; + ParallelApplyWorkerInfo *winfo; + TransApplyAction apply_action; if (in_streamed_transaction) ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg_internal("duplicate STREAM START message"))); - /* - * Start a transaction on stream start, this transaction will be committed - * on the stream stop unless it is a tablesync worker in which case it - * will be committed after processing all the messages. We need the - * transaction for handling the buffile, used for serializing the - * streaming data and subxact info. - */ - begin_replication_step(); - /* notify handle methods we're processing a remote transaction */ in_streamed_transaction = true; @@ -1203,35 +1407,80 @@ apply_handle_stream_start(StringInfo s) set_apply_error_context_xact(stream_xid, InvalidXLogRecPtr); /* - * Initialize the worker's stream_fileset if we haven't yet. This will be - * used for the entire duration of the worker so create it in a permanent - * context. We create this on the very first streaming message from any - * transaction and then use it for this and other streaming transactions. - * Now, we could create a fileset at the start of the worker as well but - * then we won't be sure that it will ever be used. + * For the first stream start, check if there is any free parallel apply + * worker we can use to process this transaction, otherwise try to start a + * new parallel apply worker. */ - if (MyLogicalRepWorker->stream_fileset == NULL) + if (first_segment) + parallel_apply_start_worker(stream_xid); + + apply_action = get_transaction_apply_action(stream_xid, &winfo); + + switch (apply_action) { - MemoryContext oldctx; + case TRANS_LEADER_SERIALIZE: - oldctx = MemoryContextSwitchTo(ApplyContext); + /* + * Start a transaction on stream start, this transaction will be + * committed on the stream stop unless it is a tablesync worker in + * which case it will be committed after processing all the + * messages. We need the transaction for handling the buffile, + * used for serializing the streaming data and subxact info. + */ + begin_replication_step(); - MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); - FileSetInit(MyLogicalRepWorker->stream_fileset); + /* + * Initialize the worker's stream_fileset if we haven't yet. This + * will be used for the entire duration of the worker so create it + * in a permanent context. We create this on the very first + * streaming message from any transaction and then use it for this + * and other streaming transactions. Now, we could create a + * fileset at the start of the worker as well but then we won't be + * sure that it will ever be used. + */ + if (MyLogicalRepWorker->stream_fileset == NULL) + { + MemoryContext oldctx; - MemoryContextSwitchTo(oldctx); - } + oldctx = MemoryContextSwitchTo(ApplyContext); - /* open the spool file for this transaction */ - stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment); + MyLogicalRepWorker->stream_fileset = palloc(sizeof(FileSet)); + FileSetInit(MyLogicalRepWorker->stream_fileset); - /* if this is not the first segment, open existing subxact file */ - if (!first_segment) - subxact_info_read(MyLogicalRepWorker->subid, stream_xid); + MemoryContextSwitchTo(oldctx); + } - pgstat_report_activity(STATE_RUNNING, NULL); + /* Open the spool file for this transaction. */ + stream_open_file(MyLogicalRepWorker->subid, stream_xid, first_segment); - end_replication_step(); + /* If this is not the first segment, open existing subxact file. */ + if (!first_segment) + subxact_info_read(MyLogicalRepWorker->subid, stream_xid); + + end_replication_step(); + break; + + case TRANS_LEADER_SEND_TO_PARALLEL: + Assert(winfo); + + parallel_apply_send_data(winfo, s->len, s->data); + + parallel_stream_nchanges = 0; + + /* Cache the parallel apply worker for this transaction. */ + stream_apply_worker = winfo; + break; + + case TRANS_PARALLEL_APPLY: + /* No special handling is required in parallel apply worker. */ + break; + + default: + Assert(false); + break; + } + + pgstat_report_activity(STATE_RUNNING, NULL); } /* @@ -1240,58 +1489,78 @@ apply_handle_stream_start(StringInfo s) static void apply_handle_stream_stop(StringInfo s) { + ParallelApplyWorkerInfo *winfo; + TransApplyAction apply_action; + if (!in_streamed_transaction) ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg_internal("STREAM STOP message without STREAM START"))); - /* - * Close the file with serialized changes, and serialize information about - * subxacts for the toplevel transaction. - */ - subxact_info_write(MyLogicalRepWorker->subid, stream_xid); - stream_close_file(); + apply_action = get_transaction_apply_action(stream_xid, &winfo); - /* We must be in a valid transaction state */ - Assert(IsTransactionState()); + switch (apply_action) + { + case TRANS_LEADER_SERIALIZE: - /* Commit the per-stream transaction */ - CommitTransactionCommand(); + /* + * Close the file with serialized changes, and serialize + * information about subxacts for the toplevel transaction. + */ + subxact_info_write(MyLogicalRepWorker->subid, stream_xid); + stream_close_file(); - in_streamed_transaction = false; + /* We must be in a valid transaction state */ + Assert(IsTransactionState()); - /* Reset per-stream context */ - MemoryContextReset(LogicalStreamingContext); + /* Commit the per-stream transaction */ + CommitTransactionCommand(); + + /* Reset per-stream context */ + MemoryContextReset(LogicalStreamingContext); + + pgstat_report_activity(STATE_IDLE, NULL); + break; + + case TRANS_LEADER_SEND_TO_PARALLEL: + Assert(winfo); + + parallel_apply_send_data(winfo, s->len, s->data); + + elog(DEBUG1, "applied %u changes in the streaming chunk", + parallel_stream_nchanges); + + stream_apply_worker = NULL; + + pgstat_report_activity(STATE_IDLE, NULL); + break; + + case TRANS_PARALLEL_APPLY: + pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL); + break; + + default: + Assert(false); + break; + } + + in_streamed_transaction = false; - pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); } /* - * Handle STREAM abort message. + * Handle STREAM ABORT message when the transaction was spilled to disk. */ static void -apply_handle_stream_abort(StringInfo s) +serialize_stream_abort(TransactionId xid, TransactionId subxid) { - TransactionId xid; - TransactionId subxid; - - if (in_streamed_transaction) - ereport(ERROR, - (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg_internal("STREAM ABORT message without STREAM STOP"))); - - logicalrep_read_stream_abort(s, &xid, &subxid); - /* * If the two XIDs are the same, it's in fact abort of toplevel xact, so * just delete the files with serialized info. */ if (xid == subxid) - { - set_apply_error_context_xact(xid, InvalidXLogRecPtr); stream_cleanup_files(MyLogicalRepWorker->subid, xid); - } else { /* @@ -1315,8 +1584,6 @@ apply_handle_stream_abort(StringInfo s) bool found = false; char path[MAXPGPATH]; - set_apply_error_context_xact(subxid, InvalidXLogRecPtr); - subidx = -1; begin_replication_step(); subxact_info_read(MyLogicalRepWorker->subid, xid); @@ -1341,7 +1608,6 @@ apply_handle_stream_abort(StringInfo s) cleanup_subxact_info(); end_replication_step(); CommitTransactionCommand(); - reset_apply_error_context_info(); return; } @@ -1364,6 +1630,96 @@ apply_handle_stream_abort(StringInfo s) end_replication_step(); CommitTransactionCommand(); } +} + +/* + * Handle STREAM ABORT message. + */ +static void +apply_handle_stream_abort(StringInfo s) +{ + TransactionId xid; + TransactionId subxid; + LogicalRepStreamAbortData abort_data; + ParallelApplyWorkerInfo *winfo; + TransApplyAction apply_action; + + if (in_streamed_transaction) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg_internal("STREAM ABORT message without STREAM STOP"))); + + /* We receive abort information only when we can apply in parallel. */ + logicalrep_read_stream_abort(s, &abort_data, + MyLogicalRepWorker->parallel_apply); + + xid = abort_data.xid; + subxid = abort_data.subxid; + + set_apply_error_context_xact(subxid, abort_data.abort_lsn); + + apply_action = get_transaction_apply_action(xid, &winfo); + + switch (apply_action) + { + case TRANS_LEADER_SERIALIZE: + + /* + * We are in the leader apply worker and the transaction has been + * serialized to file. + */ + serialize_stream_abort(xid, subxid); + break; + + case TRANS_LEADER_SEND_TO_PARALLEL: + Assert(winfo); + + if (subxid == xid) + { + /* + * The origin can be active only in one process. See + * apply_handle_stream_commit. + */ + parallel_apply_replorigin_reset(); + } + + /* Send STREAM ABORT message to the parallel apply worker. */ + parallel_apply_send_data(winfo, s->len, s->data); + + /* + * After sending the data to the parallel apply worker, wait for + * that worker to finish. This is necessary to maintain commit + * order which avoids failures due to transaction dependencies and + * deadlocks. + */ + parallel_apply_wait_for_xact_finish(winfo); + + if (subxid == xid) + { + parallel_apply_replorigin_setup(); + parallel_apply_free_worker(winfo, xid); + } + else + { + /* + * Set in_parallel_apply_xact to true again as we only aborted + * the subtransaction and the top transaction is still in + * progress. + */ + parallel_apply_set_in_xact(winfo->shared, true); + } + break; + + case TRANS_PARALLEL_APPLY: + parallel_apply_stream_abort(&abort_data); + + elog(DEBUG1, "finished processing the transaction finish command"); + break; + + default: + Assert(false); + break; + } reset_apply_error_context_info(); } @@ -1495,6 +1851,8 @@ apply_handle_stream_commit(StringInfo s) { TransactionId xid; LogicalRepCommitData commit_data; + ParallelApplyWorkerInfo *winfo; + TransApplyAction apply_action; if (in_streamed_transaction) ereport(ERROR, @@ -1504,14 +1862,79 @@ apply_handle_stream_commit(StringInfo s) xid = logicalrep_read_stream_commit(s, &commit_data); set_apply_error_context_xact(xid, commit_data.commit_lsn); - elog(DEBUG1, "received commit for streamed transaction %u", xid); + apply_action = get_transaction_apply_action(xid, &winfo); - apply_spooled_messages(xid, commit_data.commit_lsn); + switch (apply_action) + { + case TRANS_LEADER_SERIALIZE: - apply_handle_commit_internal(&commit_data); + /* + * The transaction has been serialized to file, so replay all the + * spooled operations. + */ + apply_spooled_messages(xid, commit_data.commit_lsn); + + apply_handle_commit_internal(&commit_data); + + /* Unlink the files with serialized changes and subxact info. */ + stream_cleanup_files(MyLogicalRepWorker->subid, xid); + break; + + case TRANS_LEADER_SEND_TO_PARALLEL: + Assert(winfo); + + /* + * We need to reset the replication origin before sending the + * commit message and set it up again after confirming that + * parallel worker has processed the message. This is required + * because origin can be active only in one process at-a-time. + */ + parallel_apply_replorigin_reset(); + + /* Send STREAM COMMIT message to the parallel apply worker. */ + parallel_apply_send_data(winfo, s->len, s->data); + + /* + * After sending the data to the parallel apply worker, wait for + * that worker to finish. This is necessary to maintain commit + * order which avoids failures due to transaction dependencies and + * deadlocks. + */ + parallel_apply_wait_for_xact_finish(winfo); + parallel_apply_replorigin_setup(); - /* unlink the files with serialized changes and subxact info */ - stream_cleanup_files(MyLogicalRepWorker->subid, xid); + pgstat_report_stat(false); + store_flush_position(commit_data.end_lsn); + stop_skipping_changes(); + + parallel_apply_free_worker(winfo, xid); + + /* + * The transaction is either non-empty or skipped, so we clear the + * subskiplsn. + */ + clear_subscription_skip_lsn(commit_data.commit_lsn); + break; + + case TRANS_PARALLEL_APPLY: + parallel_apply_replorigin_setup(); + + apply_handle_commit_internal(&commit_data); + + parallel_apply_replorigin_reset(); + + list_free(subxactlist); + subxactlist = NIL; + + parallel_apply_set_in_xact(MyParallelShared, false); + + elog(DEBUG1, "finished processing the transaction finish command"); + break; + + default: + Assert(false); + break; + } /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); @@ -1555,6 +1978,13 @@ apply_handle_commit_internal(LogicalRepCommitData *commit_data) replorigin_session_origin_timestamp = commit_data->committime; CommitTransactionCommand(); + + if (IsTransactionBlock()) + { + EndTransactionBlock(false); + CommitTransactionCommand(); + } + pgstat_report_stat(false); store_flush_position(commit_data->end_lsn); @@ -2492,7 +2922,7 @@ apply_handle_truncate(StringInfo s) /* * Logical replication protocol message dispatcher. */ -static void +void apply_dispatch(StringInfo s) { LogicalRepMsgType action = pq_getmsgbyte(s); @@ -2661,6 +3091,10 @@ store_flush_position(XLogRecPtr remote_lsn) { FlushPosition *flushpos; + /* Skip for parallel apply workers. */ + if (am_parallel_apply_worker()) + return; + /* Need to do this in permanent context */ MemoryContextSwitchTo(ApplyContext); @@ -2725,6 +3159,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) errcallback.callback = apply_error_callback; errcallback.previous = error_context_stack; error_context_stack = &errcallback; + apply_error_context_stack = error_context_stack; /* This outer loop iterates once per wait. */ for (;;) @@ -2939,6 +3374,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) /* Pop the error context stack */ error_context_stack = errcallback.previous; + apply_error_context_stack = error_context_stack; /* All done */ walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli); @@ -3039,7 +3475,7 @@ send_feedback(XLogRecPtr recvpos, bool force, bool requestReply) /* * Reread subscription info if needed. Most changes will be exit. */ -static void +void maybe_reread_subscription(void) { MemoryContext oldctx; @@ -3068,21 +3504,32 @@ maybe_reread_subscription(void) */ if (!newsub) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will " - "stop because the subscription was removed", - MySubscription->name))); - + if (am_parallel_apply_worker()) + ereport(LOG, + (errmsg("logical replication parallel apply worker for subscription \"%s\" will " + "stop because the subscription was removed", + MySubscription->name))); + else + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will " + "stop because the subscription was removed", + MySubscription->name))); proc_exit(0); } /* Exit if the subscription was disabled. */ if (!newsub->enabled) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will " - "stop because the subscription was disabled", - MySubscription->name))); + if (am_parallel_apply_worker()) + ereport(LOG, + (errmsg("logical replication parallel apply worker for subscription \"%s\" will " + "stop because the subscription was disabled", + MySubscription->name))); + else + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will " + "stop because the subscription was disabled", + MySubscription->name))); proc_exit(0); } @@ -3095,7 +3542,9 @@ maybe_reread_subscription(void) /* * Exit if any parameter that affects the remote connection was changed. - * The launcher will start a new worker. + * The launcher will start a new worker, but note that the parallel apply + * worker may or may not restart depending on the value of the streaming + * option and whether there will be a streaming transaction. */ if (strcmp(newsub->conninfo, MySubscription->conninfo) != 0 || strcmp(newsub->name, MySubscription->name) != 0 || @@ -3106,9 +3555,14 @@ maybe_reread_subscription(void) newsub->owner != MySubscription->owner || !equal(newsub->publications, MySubscription->publications)) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will restart because of a parameter change", - MySubscription->name))); + if (am_parallel_apply_worker()) + ereport(LOG, + (errmsg("logical replication parallel apply worker for subscription \"%s\" will stop because of a parameter change", + MySubscription->name))); + else + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will restart because of a parameter change", + MySubscription->name))); proc_exit(0); } @@ -3594,37 +4048,16 @@ start_apply(XLogRecPtr origin_startpos) PG_END_TRY(); } -/* Logical Replication Apply worker entry point */ +/* + * Common initialization for leader apply worker and parallel apply worker. + * + * Initialize the database connection, in-memory subscription and necessary + * config options. + */ void -ApplyWorkerMain(Datum main_arg) +InitializeApplyWorker(void) { - int worker_slot = DatumGetInt32(main_arg); MemoryContext oldctx; - char originname[NAMEDATALEN]; - XLogRecPtr origin_startpos = InvalidXLogRecPtr; - char *myslotname = NULL; - WalRcvStreamOptions options; - int server_version; - - /* Attach to slot */ - logicalrep_worker_attach(worker_slot); - - /* Setup signal handling */ - pqsignal(SIGHUP, SignalHandlerForConfigReload); - pqsignal(SIGTERM, die); - BackgroundWorkerUnblockSignals(); - - /* - * We don't currently need any ResourceOwner in a walreceiver process, but - * if we did, we could call CreateAuxProcessResourceOwner here. - */ - - /* Initialise stats to a sanish value */ - MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time = - MyLogicalRepWorker->reply_time = GetCurrentTimestamp(); - - /* Load the libpq-specific functions */ - load_file("libpqwalreceiver", false); /* Run as replica session replication role. */ SetConfigOption("session_replication_role", "replica", @@ -3651,10 +4084,16 @@ ApplyWorkerMain(Datum main_arg) MySubscription = GetSubscription(MyLogicalRepWorker->subid, true); if (!MySubscription) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription %u will not " - "start because the subscription was removed during startup", - MyLogicalRepWorker->subid))); + if (am_parallel_apply_worker()) + ereport(LOG, + (errmsg("logical replication parallel apply worker for subscription %u will not " + "start because the subscription was removed during startup", + MyLogicalRepWorker->subid))); + else + ereport(LOG, + (errmsg("logical replication apply worker for subscription %u will not " + "start because the subscription was removed during startup", + MyLogicalRepWorker->subid))); proc_exit(0); } @@ -3663,10 +4102,16 @@ ApplyWorkerMain(Datum main_arg) if (!MySubscription->enabled) { - ereport(LOG, - (errmsg("logical replication apply worker for subscription \"%s\" will not " - "start because the subscription was disabled during startup", - MySubscription->name))); + if (am_parallel_apply_worker()) + ereport(LOG, + (errmsg("logical replication parallel apply worker for subscription \"%s\" will not " + "start because the subscription was disabled during startup", + MySubscription->name))); + else + ereport(LOG, + (errmsg("logical replication apply worker for subscription \"%s\" will not " + "start because the subscription was disabled during startup", + MySubscription->name))); proc_exit(0); } @@ -3684,12 +4129,50 @@ ApplyWorkerMain(Datum main_arg) ereport(LOG, (errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started", MySubscription->name, get_rel_name(MyLogicalRepWorker->relid)))); + else if (am_parallel_apply_worker()) + ereport(LOG, + (errmsg("logical replication parallel apply worker for subscription \"%s\" has started", + MySubscription->name))); else ereport(LOG, (errmsg("logical replication apply worker for subscription \"%s\" has started", MySubscription->name))); CommitTransactionCommand(); +} + +/* Logical Replication Apply worker entry point */ +void +ApplyWorkerMain(Datum main_arg) +{ + int worker_slot = DatumGetInt32(main_arg); + char originname[NAMEDATALEN]; + XLogRecPtr origin_startpos = InvalidXLogRecPtr; + char *myslotname = NULL; + WalRcvStreamOptions options; + int server_version; + + /* Attach to slot */ + logicalrep_worker_attach(worker_slot); + + /* Setup signal handling */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + /* + * We don't currently need any ResourceOwner in a walreceiver process, but + * if we did, we could call CreateAuxProcessResourceOwner here. + */ + + /* Initialise stats to a sanish value */ + MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time = + MyLogicalRepWorker->reply_time = GetCurrentTimestamp(); + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + + InitializeApplyWorker(); /* Connect to the origin and start the replication. */ elog(DEBUG1, "connecting to publisher using connection string \"%s\"", @@ -3712,7 +4195,7 @@ ApplyWorkerMain(Datum main_arg) } else { - /* This is main apply worker */ + /* This is leader apply worker */ RepOriginId originid; TimeLineID startpointTLI; char *err; @@ -3777,13 +4260,36 @@ ApplyWorkerMain(Datum main_arg) server_version = walrcv_server_version(LogRepWorkerWalRcvConn); options.proto.logical.proto_version = + server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM : server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM : server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM : LOGICALREP_PROTO_VERSION_NUM; options.proto.logical.publication_names = MySubscription->publications; options.proto.logical.binary = MySubscription->binary; - options.proto.logical.streaming = MySubscription->stream; + + /* + * Assign the appropriate option value for streaming option according to + * the 'streaming' mode and the publisher's ability to support that mode. + */ + if (server_version >= 160000 && + MySubscription->stream == SUBSTREAM_PARALLEL) + { + options.proto.logical.streaming_str = pstrdup("parallel"); + MyLogicalRepWorker->parallel_apply = true; + } + else if (server_version >= 140000 && + MySubscription->stream != SUBSTREAM_OFF) + { + options.proto.logical.streaming_str = pstrdup("on"); + MyLogicalRepWorker->parallel_apply = false; + } + else + { + options.proto.logical.streaming_str = NULL; + MyLogicalRepWorker->parallel_apply = false; + } + options.proto.logical.twophase = false; options.proto.logical.origin = pstrdup(MySubscription->origin); @@ -3881,6 +4387,15 @@ IsLogicalWorker(void) } /* + * Is current process a logical replication parallel apply worker? + */ +bool +IsLogicalParallelApplyWorker(void) +{ + return IsLogicalWorker() && am_parallel_apply_worker(); +} + +/* * Start skipping changes of the transaction if the given LSN matches the * LSN specified by subscription's skiplsn. */ @@ -3942,7 +4457,7 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn) XLogRecPtr myskiplsn = MySubscription->skiplsn; bool started_tx = false; - if (likely(XLogRecPtrIsInvalid(myskiplsn))) + if (likely(XLogRecPtrIsInvalid(myskiplsn)) || am_parallel_apply_worker()) return; if (!IsTransactionState()) @@ -4014,7 +4529,7 @@ clear_subscription_skip_lsn(XLogRecPtr finish_lsn) } /* Error callback to give more context info about the change being applied */ -static void +void apply_error_callback(void *arg) { ApplyErrorCallbackArg *errarg = &apply_error_callback_arg; @@ -4042,23 +4557,47 @@ apply_error_callback(void *arg) errarg->remote_xid, LSN_FORMAT_ARGS(errarg->finish_lsn)); } - else if (errarg->remote_attnum < 0) - errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" in transaction %u, finished at %X/%X", - errarg->origin_name, - logicalrep_message_type(errarg->command), - errarg->rel->remoterel.nspname, - errarg->rel->remoterel.relname, - errarg->remote_xid, - LSN_FORMAT_ARGS(errarg->finish_lsn)); else - errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u, finished at %X/%X", - errarg->origin_name, - logicalrep_message_type(errarg->command), - errarg->rel->remoterel.nspname, - errarg->rel->remoterel.relname, - errarg->rel->remoterel.attnames[errarg->remote_attnum], - errarg->remote_xid, - LSN_FORMAT_ARGS(errarg->finish_lsn)); + { + if (errarg->remote_attnum < 0) + { + if (XLogRecPtrIsInvalid(errarg->finish_lsn)) + errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" in transaction %u", + errarg->origin_name, + logicalrep_message_type(errarg->command), + errarg->rel->remoterel.nspname, + errarg->rel->remoterel.relname, + errarg->remote_xid); + else + errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" in transaction %u, finished at %X/%X", + errarg->origin_name, + logicalrep_message_type(errarg->command), + errarg->rel->remoterel.nspname, + errarg->rel->remoterel.relname, + errarg->remote_xid, + LSN_FORMAT_ARGS(errarg->finish_lsn)); + } + else + { + if (XLogRecPtrIsInvalid(errarg->finish_lsn)) + errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u", + errarg->origin_name, + logicalrep_message_type(errarg->command), + errarg->rel->remoterel.nspname, + errarg->rel->remoterel.relname, + errarg->rel->remoterel.attnames[errarg->remote_attnum], + errarg->remote_xid); + else + errcontext("processing remote data for replication origin \"%s\" during message type \"%s\" for replication target relation \"%s.%s\" column \"%s\" in transaction %u, finished at %X/%X", + errarg->origin_name, + logicalrep_message_type(errarg->command), + errarg->rel->remoterel.nspname, + errarg->rel->remoterel.relname, + errarg->rel->remoterel.attnames[errarg->remote_attnum], + errarg->remote_xid, + LSN_FORMAT_ARGS(errarg->finish_lsn)); + } + } } /* Set transaction information of apply error callback */ @@ -4078,3 +4617,36 @@ reset_apply_error_context_info(void) apply_error_callback_arg.remote_attnum = -1; set_apply_error_context_xact(InvalidTransactionId, InvalidXLogRecPtr); } + +/* + * Return the action to take for the given transaction. *winfo is assigned to + * the destination parallel worker info (if the action is + * TRANS_LEADER_SEND_TO_PARALLEL), otherwise *winfo is assigned NULL. + */ +static TransApplyAction +get_transaction_apply_action(TransactionId xid, ParallelApplyWorkerInfo **winfo) +{ + *winfo = NULL; + + if (am_parallel_apply_worker()) + { + return TRANS_PARALLEL_APPLY; + } + else if (in_remote_transaction) + { + return TRANS_LEADER_APPLY; + } + + /* + * Check if we are processing this transaction using a parallel apply + * worker and if so, send the changes to that worker. + */ + else if ((*winfo = parallel_apply_find_worker(xid))) + { + return TRANS_LEADER_SEND_TO_PARALLEL; + } + else + { + return TRANS_LEADER_SERIALIZE; + } +} diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index 2ecaa5b..c3accf2 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -369,7 +369,7 @@ parse_output_parameters(List *options, PGOutputData *data) errmsg("conflicting or redundant options"))); streaming_given = true; - data->streaming = defGetBoolean(defel); + data->streaming = defGetStreamingMode(defel); } else if (strcmp(defel->defname, "two_phase") == 0) { @@ -461,13 +461,20 @@ pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, * we only allow it with sufficient version of the protocol, and when * the output plugin supports it. */ - if (!data->streaming) + if (data->streaming == SUBSTREAM_OFF) ctx->streaming = false; - else if (data->protocol_version < LOGICALREP_PROTO_STREAM_VERSION_NUM) + else if (data->streaming == SUBSTREAM_ON && + data->protocol_version < LOGICALREP_PROTO_STREAM_VERSION_NUM) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("requested proto_version=%d does not support streaming, need %d or higher", data->protocol_version, LOGICALREP_PROTO_STREAM_VERSION_NUM))); + else if (data->streaming == SUBSTREAM_PARALLEL && + data->protocol_version < LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("requested proto_version=%d does not support parallel streaming, need %d or higher", + data->protocol_version, LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM))); else if (!ctx->streaming) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -513,7 +520,7 @@ pgoutput_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, * Disable the streaming and prepared transactions during the slot * initialization mode. */ - ctx->streaming = false; + ctx->streaming = SUBSTREAM_OFF; ctx->twophase = false; } } @@ -1839,6 +1846,8 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx, XLogRecPtr abort_lsn) { ReorderBufferTXN *toptxn; + PGOutputData *data = (PGOutputData *) ctx->output_plugin_private; + bool write_abort_info = (data->streaming == SUBSTREAM_PARALLEL); /* * The abort should happen outside streaming block, even for streamed @@ -1852,7 +1861,9 @@ pgoutput_stream_abort(struct LogicalDecodingContext *ctx, Assert(rbtxn_is_streamed(toptxn)); OutputPluginPrepareWrite(ctx, true); - logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn->xid); + logicalrep_write_stream_abort(ctx->out, toptxn->xid, txn->xid, abort_lsn, + txn->xact_time.abort_time, write_abort_info); + OutputPluginWrite(ctx, true); cleanup_rel_sync_cache(toptxn->xid, false); diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 7767657..7c009d5 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -22,6 +22,7 @@ #include "commands/async.h" #include "miscadmin.h" #include "pgstat.h" +#include "replication/logicalworker.h" #include "replication/walsender.h" #include "storage/condition_variable.h" #include "storage/ipc.h" @@ -657,6 +658,9 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_LOG_MEMORY_CONTEXT)) HandleLogMemoryContextInterrupt(); + if (CheckProcSignal(PROCSIG_PARALLEL_APPLY_MESSAGE)) + HandleParallelApplyMessageInterrupt(); + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_DATABASE)) RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_DATABASE); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index a9a1851..f1aa795 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3379,6 +3379,9 @@ ProcessInterrupts(void) if (LogMemoryContextPending) ProcessLogMemoryContextInterrupt(); + + if (ParallelApplyMessagePending) + HandleParallelApplyMessages(); } /* diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index 92f24a6..dcff598 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -230,6 +230,9 @@ pgstat_get_wait_activity(WaitEventActivity w) case WAIT_EVENT_LOGICAL_LAUNCHER_MAIN: event_name = "LogicalLauncherMain"; break; + case WAIT_EVENT_LOGICAL_PARALLEL_APPLY_MAIN: + event_name = "LogicalParallelApplyMain"; + break; case WAIT_EVENT_RECOVERY_WAL_STREAM: event_name = "RecoveryWalStream"; break; @@ -388,6 +391,9 @@ pgstat_get_wait_ipc(WaitEventIPC w) case WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT: event_name = "HashGrowBucketsReinsert"; break; + case WAIT_EVENT_LOGICAL_PARALLEL_APPLY_STATE_CHANGE: + event_name = "LogicalParallelApplyStateChange"; + break; case WAIT_EVENT_LOGICAL_SYNC_DATA: event_name = "LogicalSyncData"; break; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 05ab087..f3de4ff 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -2964,6 +2964,18 @@ struct config_int ConfigureNamesInt[] = }, { + {"max_parallel_apply_workers_per_subscription", + PGC_SIGHUP, + REPLICATION_SUBSCRIBERS, + gettext_noop("Maximum number of parallel apply workers per subscription."), + NULL, + }, + &max_parallel_apply_workers_per_subscription, + 2, 0, MAX_BACKENDS, + NULL, NULL, NULL + }, + + { {"log_rotation_age", PGC_SIGHUP, LOGGING_WHERE, gettext_noop("Sets the amount of time to wait before forcing " "log file rotation."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 868d21c..4d01ca9 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -360,6 +360,7 @@ #max_logical_replication_workers = 4 # taken from max_worker_processes # (change requires restart) #max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers +#max_parallel_apply_workers_per_subscription = 2 # taken from max_logical_replication_workers #------------------------------------------------------------------------------ diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index bd9b066..01b2b60 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -4481,7 +4481,7 @@ getSubscriptions(Archive *fout) if (fout->remoteVersion >= 140000) appendPQExpBufferStr(query, " s.substream,\n"); else - appendPQExpBufferStr(query, " false AS substream,\n"); + appendPQExpBufferStr(query, " 'f' AS substream,\n"); if (fout->remoteVersion >= 150000) appendPQExpBufferStr(query, @@ -4618,8 +4618,10 @@ dumpSubscription(Archive *fout, const SubscriptionInfo *subinfo) if (strcmp(subinfo->subbinary, "t") == 0) appendPQExpBufferStr(query, ", binary = true"); - if (strcmp(subinfo->substream, "f") != 0) + if (strcmp(subinfo->substream, "t") == 0) appendPQExpBufferStr(query, ", streaming = on"); + else if (strcmp(subinfo->substream, "p") == 0) + appendPQExpBufferStr(query, ", streaming = parallel"); if (strcmp(subinfo->subtwophasestate, two_phase_disabled) != 0) appendPQExpBufferStr(query, ", two_phase = on"); diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index 7b98714..016afbc 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -80,7 +80,8 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW bool subbinary; /* True if the subscription wants the * publisher to send data in binary */ - bool substream; /* Stream in-progress transactions. */ + char substream; /* Stream in-progress transactions. See + * SUBSTREAM_xxx constants. */ char subtwophasestate; /* Stream two-phase transactions */ @@ -124,7 +125,8 @@ typedef struct Subscription bool enabled; /* Indicates if the subscription is enabled */ bool binary; /* Indicates if the subscription wants data in * binary format */ - bool stream; /* Allow streaming in-progress transactions. */ + char stream; /* Allow streaming in-progress transactions. + * See SUBSTREAM_xxx constants. */ char twophasestate; /* Allow streaming two-phase transactions */ bool disableonerr; /* Indicates if the subscription should be * automatically disabled if a worker error @@ -137,6 +139,21 @@ typedef struct Subscription * specified origin */ } Subscription; +/* Disallow streaming in-progress transactions. */ +#define SUBSTREAM_OFF 'f' + +/* + * Streaming in-progress transactions are written to a temporary file and + * applied only after the transaction is committed on upstream. + */ +#define SUBSTREAM_ON 't' + +/* + * Streaming in-progress transactions are applied immediately via a parallel + * apply worker. + */ +#define SUBSTREAM_PARALLEL 'p' + extern Subscription *GetSubscription(Oid subid, bool missing_ok); extern void FreeSubscription(Subscription *sub); extern void DisableSubscription(Oid subid); diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 56d2bb6..f6ba5ff 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -154,6 +154,7 @@ extern List *defGetQualifiedName(DefElem *def); extern TypeName *defGetTypeName(DefElem *def); extern int defGetTypeLength(DefElem *def); extern List *defGetStringList(DefElem *def); +extern char defGetStreamingMode(DefElem *def); extern void errorConflictingDefElem(DefElem *defel, ParseState *pstate) pg_attribute_noreturn(); #endif /* DEFREM_H */ diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index f1e2821..d513ef5 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -14,6 +14,7 @@ extern PGDLLIMPORT int max_logical_replication_workers; extern PGDLLIMPORT int max_sync_workers_per_subscription; +extern PGDLLIMPORT int max_parallel_apply_workers_per_subscription; extern void ApplyLauncherRegister(void); extern void ApplyLauncherMain(Datum main_arg); diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h index 7eaa4c9..1da7b01 100644 --- a/src/include/replication/logicalproto.h +++ b/src/include/replication/logicalproto.h @@ -32,12 +32,17 @@ * * LOGICALREP_PROTO_TWOPHASE_VERSION_NUM is the minimum protocol version with * support for two-phase commit decoding (at prepare time). Introduced in PG15. + * + * LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM is the minimum protocol version + * where we support applying large streaming transactions in parallel. + * Introduced in PG16. */ #define LOGICALREP_PROTO_MIN_VERSION_NUM 1 #define LOGICALREP_PROTO_VERSION_NUM 1 #define LOGICALREP_PROTO_STREAM_VERSION_NUM 2 #define LOGICALREP_PROTO_TWOPHASE_VERSION_NUM 3 -#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_TWOPHASE_VERSION_NUM +#define LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM 4 +#define LOGICALREP_PROTO_MAX_VERSION_NUM LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM /* * Logical message types @@ -175,6 +180,17 @@ typedef struct LogicalRepRollbackPreparedTxnData char gid[GIDSIZE]; } LogicalRepRollbackPreparedTxnData; +/* + * Transaction protocol information for stream abort. + */ +typedef struct LogicalRepStreamAbortData +{ + TransactionId xid; + TransactionId subxid; + XLogRecPtr abort_lsn; + TimestampTz abort_time; +} LogicalRepStreamAbortData; + extern void logicalrep_write_begin(StringInfo out, ReorderBufferTXN *txn); extern void logicalrep_read_begin(StringInfo in, LogicalRepBeginData *begin_data); @@ -246,9 +262,13 @@ extern void logicalrep_write_stream_commit(StringInfo out, ReorderBufferTXN *txn extern TransactionId logicalrep_read_stream_commit(StringInfo in, LogicalRepCommitData *commit_data); extern void logicalrep_write_stream_abort(StringInfo out, TransactionId xid, - TransactionId subxid); -extern void logicalrep_read_stream_abort(StringInfo in, TransactionId *xid, - TransactionId *subxid); + TransactionId subxid, + XLogRecPtr abort_lsn, + TimestampTz abort_time, + bool write_abort_info); +extern void logicalrep_read_stream_abort(StringInfo in, + LogicalRepStreamAbortData *abort_data, + bool read_abort_info); extern char *logicalrep_message_type(LogicalRepMsgType action); #endif /* LOGICAL_PROTO_H */ diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index cd1b6e8..dc68cd8 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -12,8 +12,16 @@ #ifndef LOGICALWORKER_H #define LOGICALWORKER_H +#include + +extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; + extern void ApplyWorkerMain(Datum main_arg); +extern void ParallelApplyWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); +extern bool IsLogicalParallelApplyWorker(void); +extern void HandleParallelApplyMessageInterrupt(void); +extern void HandleParallelApplyMessages(void); #endif /* LOGICALWORKER_H */ diff --git a/src/include/replication/pgoutput.h b/src/include/replication/pgoutput.h index 0202755..3c30da8 100644 --- a/src/include/replication/pgoutput.h +++ b/src/include/replication/pgoutput.h @@ -26,7 +26,7 @@ typedef struct PGOutputData List *publication_names; List *publications; bool binary; - bool streaming; + char streaming; bool messages; bool two_phase; char *origin; diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h index 02b59a1..1549116 100644 --- a/src/include/replication/reorderbuffer.h +++ b/src/include/replication/reorderbuffer.h @@ -301,6 +301,7 @@ typedef struct ReorderBufferTXN { TimestampTz commit_time; TimestampTz prepare_time; + TimestampTz abort_time; } xact_time; /* @@ -664,9 +665,11 @@ extern void ReorderBufferAssignChild(ReorderBuffer *rb, TransactionId xid, extern void ReorderBufferCommitChild(ReorderBuffer *rb, TransactionId xid, TransactionId subxid, XLogRecPtr commit_lsn, XLogRecPtr end_lsn); -extern void ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn); +extern void ReorderBufferAbort(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn, + TimestampTz abort_time); extern void ReorderBufferAbortOld(ReorderBuffer *rb, TransactionId oldestRunningXid); -extern void ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn); +extern void ReorderBufferForget(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn, + TimestampTz abort_time); extern void ReorderBufferInvalidate(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn); extern void ReorderBufferSetBaseSnapshot(ReorderBuffer *rb, TransactionId xid, diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 9339f29..0c8bbdd 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -182,7 +182,7 @@ typedef struct uint32 proto_version; /* Logical protocol version */ List *publication_names; /* String list of publications */ bool binary; /* Ask publisher to use binary */ - bool streaming; /* Streaming of large transactions */ + char *streaming_str; /* Streaming of large transactions */ bool twophase; /* Streaming of two-phase transactions at * prepare time */ char *origin; /* Only publish data originating from the diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 2b7114f..87f2ec9 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -17,8 +17,12 @@ #include "access/xlogdefs.h" #include "catalog/pg_subscription.h" #include "datatype/timestamp.h" +#include "miscadmin.h" +#include "replication/logicalrelation.h" #include "storage/fileset.h" #include "storage/lock.h" +#include "storage/shm_mq.h" +#include "storage/shm_toc.h" #include "storage/spin.h" @@ -60,6 +64,15 @@ typedef struct LogicalRepWorker */ FileSet *stream_fileset; + /* + * PID of leader apply worker if this slot is used for a parallel apply + * worker, InvalidPid otherwise. + */ + pid_t apply_leader_pid; + + /* Indicates whether apply can be performed in parallel. */ + bool parallel_apply; + /* Stats. */ XLogRecPtr last_lsn; TimestampTz last_send_time; @@ -68,9 +81,81 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* Struct for saving and restoring apply errcontext information */ +typedef struct ApplyErrorCallbackArg +{ + LogicalRepMsgType command; /* 0 if invalid */ + LogicalRepRelMapEntry *rel; + + /* Remote node information */ + int remote_attnum; /* -1 if invalid */ + TransactionId remote_xid; + XLogRecPtr finish_lsn; + char *origin_name; +} ApplyErrorCallbackArg; + +/* + * Struct for sharing information between leader apply worker and parallel + * apply workers. + */ +typedef struct ParallelApplyWorkerShared +{ + slock_t mutex; + + /* + * Flag used to ensure commit ordering. + * + * The parallel apply worker will set it to false after handling the + * transaction finish commands while the apply leader will wait for it to + * become false before proceeding in transaction finish commands (e.g. + * STREAM_COMMIT/STREAM_ABORT/STREAM_PREPARE). + */ + bool in_parallel_apply_xact; + + /* Information from the corresponding LogicalRepWorker slot. */ + uint16 logicalrep_worker_generation; + + int logicalrep_worker_slot_no; +} ParallelApplyWorkerShared; + +/* + * Information which is used to manage the parallel apply worker. + */ +typedef struct ParallelApplyWorkerInfo +{ + shm_mq_handle *mq_handle; + + /* + * The queue used to transfer messages from the parallel apply worker to + * the leader apply worker. + */ + shm_mq_handle *error_mq_handle; + + dsm_segment *dsm_seg; + + /* + * True if the worker is being used to process a parallel apply + * transaction. False indicates this worker is available for re-use. + */ + bool in_use; + + ParallelApplyWorkerShared *shared; +} ParallelApplyWorkerInfo; + /* Main memory context for apply worker. Permanent during worker lifetime. */ extern PGDLLIMPORT MemoryContext ApplyContext; +extern PGDLLIMPORT MemoryContext ApplyMessageContext; + +extern PGDLLIMPORT ErrorContextCallback *apply_error_context_stack; +extern PGDLLIMPORT ApplyErrorCallbackArg apply_error_callback_arg; + +extern PGDLLIMPORT bool MySubscriptionValid; + +extern PGDLLIMPORT ParallelApplyWorkerShared *MyParallelShared; + +extern PGDLLIMPORT List *subxactlist; + /* libpqreceiver connection */ extern PGDLLIMPORT struct WalReceiverConn *LogRepWorkerWalRcvConn; @@ -79,18 +164,23 @@ extern PGDLLIMPORT Subscription *MySubscription; extern PGDLLIMPORT LogicalRepWorker *MyLogicalRepWorker; extern PGDLLIMPORT bool in_remote_transaction; +extern PGDLLIMPORT bool in_streamed_transaction; +extern PGDLLIMPORT ParallelApplyWorkerInfo *stream_apply_worker; extern void logicalrep_worker_attach(int slot); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running); -extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, - Oid userid, Oid relid); +extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, + Oid userid, Oid relid, + dsm_handle subworker_dsm); extern void logicalrep_worker_stop(Oid subid, Oid relid); +extern void logicalrep_worker_stop_by_slot(int slot_no, uint16 generation); extern void logicalrep_worker_wakeup(Oid subid, Oid relid); extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker); extern int logicalrep_sync_worker_count(Oid subid); +extern int logicalrep_parallel_apply_worker_count(Oid subid); extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); @@ -103,10 +193,50 @@ extern void process_syncing_tables(XLogRecPtr current_lsn); extern void invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue); +extern void apply_dispatch(StringInfo s); + +extern void maybe_reread_subscription(void); + +extern void InitializeApplyWorker(void); + +/* Function for apply error callback */ +extern void apply_error_callback(void *arg); + +/* Parallel apply worker setup and interactions */ +extern void parallel_apply_start_worker(TransactionId xid); +extern ParallelApplyWorkerInfo *parallel_apply_find_worker(TransactionId xid); +extern void parallel_apply_set_in_xact(ParallelApplyWorkerShared *wshared, + bool in_xact); +extern void parallel_apply_free_worker(ParallelApplyWorkerInfo *winfo, + TransactionId xid); +extern void parallel_apply_wait_for_xact_finish(ParallelApplyWorkerInfo *winfo); +extern void parallel_apply_send_data(ParallelApplyWorkerInfo *winfo, Size nbytes, + const void *data); + +extern void parallel_apply_start_subtrans(TransactionId current_xid, TransactionId top_xid); +extern void parallel_apply_stream_abort(LogicalRepStreamAbortData *abort_data); +extern void parallel_apply_replorigin_setup(void); +extern void parallel_apply_replorigin_reset(void); + +#define isParallelApplyWorker(worker) ((worker)->apply_leader_pid != InvalidPid) + static inline bool am_tablesync_worker(void) { return OidIsValid(MyLogicalRepWorker->relid); } +static inline bool +am_leader_apply_worker(void) +{ + return (!OidIsValid(MyLogicalRepWorker->relid) && + !isParallelApplyWorker(MyLogicalRepWorker)); +} + +static inline bool +am_parallel_apply_worker(void) +{ + return isParallelApplyWorker(MyLogicalRepWorker); +} + #endif /* WORKER_INTERNAL_H */ diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index ee63690..93a51f4 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -35,6 +35,7 @@ typedef enum PROCSIG_WALSND_INIT_STOPPING, /* ask walsenders to prepare for shutdown */ PROCSIG_BARRIER, /* global barrier interrupt */ PROCSIG_LOG_MEMORY_CONTEXT, /* ask backend to log the memory contexts */ + PROCSIG_PARALLEL_APPLY_MESSAGE, /* Message from parallel apply workers */ /* Recovery conflict reasons */ PROCSIG_RECOVERY_CONFLICT_DATABASE, diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index 6f2d561..c651ad2 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -42,6 +42,7 @@ typedef enum WAIT_EVENT_CHECKPOINTER_MAIN, WAIT_EVENT_LOGICAL_APPLY_MAIN, WAIT_EVENT_LOGICAL_LAUNCHER_MAIN, + WAIT_EVENT_LOGICAL_PARALLEL_APPLY_MAIN, WAIT_EVENT_RECOVERY_WAL_STREAM, WAIT_EVENT_SYSLOGGER_MAIN, WAIT_EVENT_WAL_RECEIVER_MAIN, @@ -105,6 +106,7 @@ typedef enum WAIT_EVENT_HASH_GROW_BUCKETS_ALLOCATE, WAIT_EVENT_HASH_GROW_BUCKETS_ELECT, WAIT_EVENT_HASH_GROW_BUCKETS_REINSERT, + WAIT_EVENT_LOGICAL_PARALLEL_APPLY_STATE_CHANGE, WAIT_EVENT_LOGICAL_SYNC_DATA, WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE, WAIT_EVENT_MQ_INTERNAL, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index c13d218..0f65cc7 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -255,9 +255,9 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); (1 row) DROP SUBSCRIPTION regress_testsub; --- fail - streaming must be boolean +-- fail - streaming must be boolean or 'parallel' CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo); -ERROR: streaming requires a Boolean value +ERROR: streaming requires a Boolean value or "parallel" -- now it works CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true); WARNING: subscription was created, but is not connected @@ -269,6 +269,14 @@ HINT: To initiate replication, you must manually create the replication slot, e regress_testsub | regress_subscription_user | f | {testpub} | f | t | d | f | any | off | dbname=regress_doesnotexist | 0/0 (1 row) +ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Binary | Streaming | Two-phase commit | Disable on error | Origin | Synchronous commit | Conninfo | Skip LSN +-----------------+---------------------------+---------+-------------+--------+-----------+------------------+------------------+--------+--------------------+-----------------------------+---------- + regress_testsub | regress_subscription_user | f | {testpub} | f | p | d | f | any | off | dbname=regress_doesnotexist | 0/0 +(1 row) + ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); \dRs+ diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index eaeade8..7991abf 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -167,7 +167,7 @@ ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); DROP SUBSCRIPTION regress_testsub; --- fail - streaming must be boolean +-- fail - streaming must be boolean or 'parallel' CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo); -- now it works @@ -175,6 +175,10 @@ CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUB \dRs+ +ALTER SUBSCRIPTION regress_testsub SET (streaming = parallel); + +\dRs+ + ALTER SUBSCRIPTION regress_testsub SET (streaming = false); ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index d9b839c..26ce8be 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1472,6 +1472,7 @@ LogicalRepRelId LogicalRepRelMapEntry LogicalRepRelation LogicalRepRollbackPreparedTxnData +LogicalRepStreamAbortData LogicalRepTupleData LogicalRepTyp LogicalRepWorker @@ -1663,6 +1664,9 @@ OverrideStackEntry OverridingKind PACE_HEADER PACL +ParallelApplyWorkerEntry +ParallelApplyWorkerInfo +ParallelApplyWorkerShared PATH PBOOL PCtxtHandle @@ -2784,6 +2788,7 @@ TransactionStmtKind TransformInfo TransformJsonStringValuesState TransitionCaptureState +TransApplyAction TrgmArc TrgmArcInfo TrgmBound -- 2.7.2.windows.1