From 36ed3882145e93a309eb4296879bba26b079c3ef Mon Sep 17 00:00:00 2001
From: Vigneshwaran C <vignesh21@gmail.com>
Date: Tue, 31 May 2022 11:53:32 +0530
Subject: [PATCH v19 4/4] Document bidirectional logical replication steps in
 various scenarios.

Document the steps for the following:
a) Create a two-node bidirectional replication when there is no data in both the
nodes.
b) Adding a new node when there is no data in any of the nodes.
c) Adding a new node when data is present in the existing nodes.
d) Generic steps to add a new node to the existing set of nodes.
---
 doc/src/sgml/logical-replication.sgml     | 310 ++++++++++++++++++++++
 doc/src/sgml/ref/create_subscription.sgml |   5 +-
 2 files changed, 314 insertions(+), 1 deletion(-)

diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml
index 145ea71d61..27867c0aa0 100644
--- a/doc/src/sgml/logical-replication.sgml
+++ b/doc/src/sgml/logical-replication.sgml
@@ -1267,4 +1267,314 @@ CREATE SUBSCRIPTION mysub CONNECTION 'dbname=foo host=bar user=repuser' PUBLICAT
    incremental changes to those tables.
   </para>
  </sect1>
+
+ <sect1 id="logical-replication-bidirectional">
+  <title>Bidirectional logical replication</title>
+
+   <para>
+    Bidirectional replication is useful in creating a multi-master database
+    which helps in performing read/write operations from any of the nodes.
+    The steps to create a bidirectional replication in various scenarios are
+    given below.
+   </para>
+
+   <warning>
+    <para>
+     Setting up bidirectional logical replication across nodes requires multiple
+     steps to be performed on various nodes. Because not all operations are
+     transactional, the user is advised to take backups.
+    </para>
+   </warning>
+
+  <sect2 id="setting-bidirectional-replication-two-nodes">
+   <title>Setting bidirectional replication between two nodes</title>
+   <para>
+    The steps to create a two-node bidirectional replication when there is no
+    data in both the nodes <literal>node1</literal> and
+    <literal>node2</literal> are given below:
+   </para>
+
+   <para>
+    Create a publication in <literal>node1</literal>:
+<programlisting>
+node1=# CREATE PUBLICATION pub_node1 FOR TABLE t1;
+CREATE PUBLICATION
+</programlisting></para>
+
+   <para>
+    Create a publication in <literal>node2</literal>:
+<programlisting>
+node2=# CREATE PUBLICATION pub_node2 FOR TABLE t1;
+CREATE PUBLICATION
+</programlisting></para>
+
+   <para>
+    Lock the required tables of <literal>node1</literal> and
+    <literal>node2</literal> in <literal>EXCLUSIVE</literal> mode until the
+    setup is completed.
+   </para>
+
+   <para>
+    Create a subscription in <literal>node2</literal> to subscribe to
+    <literal>node1</literal>:
+<programlisting>
+node2=# CREATE SUBSCRIPTION sub_node2_node1
+node2-# CONNECTION 'dbname=foo host=node1 user=repuser'
+node2-# PUBLICATION pub_node1
+node2-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Create a subscription in <literal>node1</literal> to subscribe to
+    <literal>node2</literal>:
+<programlisting>
+node1=# CREATE SUBSCRIPTION sub_node1_node2
+node1-# CONNECTION 'dbname=foo host=node2 user=repuser'
+node1-# PUBLICATION pub_node2
+node1-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Now the bidirectional logical replication setup is complete between
+    <literal>node1</literal> and <literal>node2</literal>. Any incremental
+    changes from <literal>node1</literal> will be replicated to
+    <literal>node2</literal> and the incremental changes from
+    <literal>node2</literal> will be replicated to <literal>node1</literal>.
+   </para>
+  </sect2>
+
+  <sect2 id="add-new-node">
+   <title>Adding a new node when there is no data in any of the nodes</title>
+   <para>
+    Adding a new node <literal>node3</literal> to the existing
+    <literal>node1</literal> and <literal>node2</literal> when there is no data
+    in any of the nodes requires setting up subscription in
+    <literal>node1</literal> and <literal>node2</literal> to replicate the data
+    from <literal>node3</literal> and setting up subscription in
+    <literal>node3</literal> to replicate data from <literal>node1</literal>
+    and <literal>node2</literal>.
+   </para>
+
+   <note>
+    <para>
+     It is assumed that the bidirectional logical replication between
+     <literal>node1</literal> and <literal>node2</literal> is already completed.
+    </para>
+   </note>
+
+   <para>
+    Create a publication in <literal>node3</literal>:
+<programlisting>
+node3=# CREATE PUBLICATION pub_node3 FOR TABLE t1;
+CREATE PUBLICATION
+</programlisting></para>
+
+   <para>
+    Lock the required tables of all the nodes <literal>node1</literal>,
+    <literal>node2</literal> and <literal>node3</literal> in
+    <literal>EXCLUSIVE</literal> mode until the setup is completed.
+   </para>
+
+   <para>
+    Create a subscription in <literal>node1</literal> to subscribe to
+    <literal>node3</literal>:
+<programlisting>
+node1=# CREATE SUBSCRIPTION sub_node1_node3
+node1-# CONNECTION 'dbname=foo host=node3 user=repuser'
+node1-# PUBLICATION pub_node3
+node1-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Create a subscription in <literal>node2</literal> to subscribe to
+    <literal>node3</literal>:
+<programlisting>
+node2=# CREATE SUBSCRIPTION sub_node2_node3
+node2-# CONNECTION 'dbname=foo host=node3 user=repuser'
+node2-# PUBLICATION pub_node3
+node2-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Create a subscription in <literal>node3</literal> to subscribe to
+    <literal>node1</literal>:
+<programlisting>
+node3=# CREATE SUBSCRIPTION sub_node3_node1
+node3-# CONNECTION 'dbname=foo host=node1 user=repuser'
+node3-# PUBLICATION pub_node1
+node3-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Create a subscription in <literal>node3</literal> to subscribe to
+    <literal>node2</literal>:
+<programlisting>
+node3=# CREATE SUBSCRIPTION sub_node3_node2
+node3-# CONNECTION 'dbname=foo host=node2 user=repuser'
+node3-# PUBLICATION pub_node2
+node3-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Now the bidirectional logical replication setup is complete between
+    <literal>node1</literal>, <literal>node2</literal> and
+    <literal>node3</literal>. Any subsequent changes in one node will
+    replicate the changes to the other nodes.
+   </para>
+  </sect2>
+
+  <sect2 id="add-new-node-data-in-existing-node">
+   <title>Adding a new node when data is present in the existing nodes</title>
+    <para>
+     Adding a new node <literal>node3</literal> which has no data to the
+     existing <literal>node1</literal> and <literal>node2</literal> when data
+     is present in existing nodes <literal>node1</literal> and
+     <literal>node2</literal> needs similar steps. The only change required
+     here is that <literal>node3</literal> should create a subscription with
+     <literal>copy_data = force</literal> to one of the existing nodes to
+     receive the existing data during initial data synchronization.
+   </para>
+
+   <note>
+    <para>
+     It is assumed that the bidirectional logical replication between
+     <literal>node1</literal> and <literal>node2</literal> is already completed.
+     The nodes <literal>node1</literal> and <literal>node2</literal> has some
+     pre-existing data in table t1 that is synchronized in both the nodes.
+    </para>
+   </note>
+
+   <para>
+    Create a publication in <literal>node3</literal>:
+<programlisting>
+node3=# CREATE PUBLICATION pub_node3 FOR TABLE t1;
+CREATE PUBLICATION
+</programlisting></para>
+
+   <para>
+    Lock the required tables of <literal>node2</literal> and
+    <literal>node3</literal> in <literal>EXCLUSIVE</literal> mode until the
+    setup is completed. No need to lock the tables in <literal>node1</literal>
+    as any data changes made will be synchronized while creating the
+    subscription with <literal>copy_data</literal> specified as
+    <literal>force</literal>.
+   </para>
+
+   <para>
+    Create a subscription in <literal>node1</literal> to subscribe to
+    <literal>node3</literal>:
+<programlisting>
+node1=# CREATE SUBSCRIPTION sub_node1_node3
+node1-# CONNECTION 'dbname=foo host=node3 user=repuser'
+node1-# PUBLICATION pub_node3
+node1-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Create a subscription in <literal>node2</literal> to subscribe to
+    <literal>node3</literal>:
+<programlisting>
+node2=# CREATE SUBSCRIPTION sub_node2_node3
+node2-# CONNECTION 'dbname=foo host=node3 user=repuser'
+node2-# PUBLICATION pub_node3
+node2-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Create a subscription in <literal>node3</literal> to subscribe to
+    <literal>node1</literal>. Use <literal>copy_data</literal> specified as
+    <literal>force</literal> so that the existing table data is
+    copied during initial sync:
+<programlisting>
+node3=# CREATE SUBSCRIPTION sub_node3_node1
+node3-# CONNECTION 'dbname=foo host=node1 user=repuser'
+node3-# PUBLICATION pub_node1
+node3-# WITH (copy_data = force, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Create a subscription in <literal>node3</literal> to subscribe to
+    <literal>node2</literal>. Use <literal>copy_data</literal> specified as
+    <literal>off</literal> because the initial table data would have been
+    already copied in the previous step:
+<programlisting>
+node3=# CREATE SUBSCRIPTION sub_node3_node2
+node3-# CONNECTION 'dbname=foo host=node2 user=repuser'
+node3-# PUBLICATION pub_node2
+node3-# WITH (copy_data = off, origin = local);
+CREATE SUBSCRIPTION
+</programlisting></para>
+
+   <para>
+    Now the bidirectional logical replication setup is complete between
+    <literal>node1</literal>, <literal>node2</literal> and
+    <literal>node3</literal>. Any subsequent changes in one node will
+    replicate the changes to the other nodes.
+   </para>
+  </sect2>
+
+  <sect2 id="add-node-data-present-in-new-node">
+   <title>Adding a new node when data is present in the new node</title>
+   <warning>
+    <para>
+     Adding a new node <literal>node3</literal> to the existing
+     <literal>node1</literal> and <literal>node2</literal> when data is present
+     in the new node <literal>node3</literal> is not possible.
+    </para>
+   </warning>
+  </sect2>
+
+  <sect2 id="generic-steps-add-new-node">
+   <title>Generic steps to add a new node to the existing set of nodes</title>
+   <para>
+    1. Create the required publication on the new node.
+   </para>
+   <para>
+    2. Lock the required tables of the new node in <literal>EXCLUSIVE</literal>
+    mode until the setup is complete. This is required to prevent any
+    modifications from happening in the new node. If data modifications occur
+    after step-3, there is a chance that the modifications will be published to
+    the first node and then synchronized back to the new node while creating
+    subscription in step-5 resulting in inconsistent data.
+   </para>
+   <para>
+    3. Create subscriptions on existing nodes pointing to publication on
+    the new node with <literal>origin</literal> parameter specified as
+    <literal>local</literal> and <literal>copy_data</literal> specified as
+    <literal>off</literal>.
+   </para>
+   <para>
+    4. Lock the required tables of the existing nodes except the first node in
+    <literal>EXCLUSIVE</literal> mode until the setup is complete. This is
+    required to prevent any modifications from happening. If data modifications
+    occur, there is a chance that the modifications done in between step-5 and
+    step-6 will not be synchronized to the new node resulting in inconsistent
+    data. No need to lock the tables in the first node as any data changes
+    made will be synchronized while creating the subscription with
+    <literal>copy_data</literal> specified as <literal>force</literal>.
+   </para>
+   <para>
+    5. Create a subscription on the new node pointing to publication on the
+    first node with <literal>origin</literal> parameter specified as
+    <literal>local</literal> and <literal>copy_data</literal> parameter
+    specified as <literal>force</literal>.
+   </para>
+   <para>
+    6. Create subscriptions on the new node pointing to publications on the
+    remaining nodes with <literal>origin</literal> parameter specified as
+    <literal>local</literal> and <literal>copy_data</literal> parameter
+    specifiedas <literal>off</literal>.
+   </para>
+  </sect2>
+ </sect1>
+
 </chapter>
diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml
index d2285b4dbd..e7b1229dca 100644
--- a/doc/src/sgml/ref/create_subscription.sgml
+++ b/doc/src/sgml/ref/create_subscription.sgml
@@ -403,7 +403,10 @@ CREATE SUBSCRIPTION <replaceable class="parameter">subscription_name</replaceabl
    being subscribed to any other publisher and throw an error to prevent
    inconsistent data in the subscription. The user can continue with the copy
    operation without throwing any error in this case by specifying
-   <literal>copy_data = force</literal>.
+   <literal>copy_data = force</literal>. Refer to the
+   <xref linkend="logical-replication-bidirectional"/> on how
+   <literal>copy_data</literal> and <literal>origin</literal> can be used
+   in bidirectional replication.
   </para>
 
  </refsect1>
-- 
2.32.0

