From a3bf8c6819875330955cab8f68636c351db837ea Mon Sep 17 00:00:00 2001
From: "kuroda.hayato%40jp.fujitsu.com" <kuroda.hayato@jp.fujitsu.com>
Date: Wed, 18 May 2022 04:56:18 +0000
Subject: [PATCH v3 3/3] add documents

---
 doc/src/sgml/catalogs.sgml | 328 +++++++++++++++++++++++++++++
 doc/src/sgml/filelist.sgml |   1 +
 doc/src/sgml/func.sgml     | 113 ++++++++++
 doc/src/sgml/lrg.sgml      | 417 +++++++++++++++++++++++++++++++++++++
 doc/src/sgml/postgres.sgml |   1 +
 5 files changed, 860 insertions(+)
 create mode 100644 doc/src/sgml/lrg.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index a533a2153e..db45b339e8 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -200,6 +200,26 @@
       <entry>metadata for large objects</entry>
      </row>
 
+     <row>
+      <entry><link linkend="catalog-pg-lrg-info"><structname>pg_lrg_info</structname></link></entry>
+      <entry>logical replication group</entry>
+     </row>
+
+     <row>
+      <entry><link linkend="catalog-pg-lrg-nodes"><structname>pg_lrg_nodes</structname></link></entry>
+      <entry>logical replication group</entry>
+     </row>
+
+     <row>
+      <entry><link linkend="catalog-pg-lrg-pub"><structname>pg_lrg_pub</structname></link></entry>
+      <entry>logical replication group</entry>
+     </row>
+
+     <row>
+      <entry><link linkend="catalog-pg-lrg-sub"><structname>pg_lrg_sub</structname></link></entry>
+      <entry>logical replication group</entry>
+     </row>
+
      <row>
       <entry><link linkend="catalog-pg-namespace"><structname>pg_namespace</structname></link></entry>
       <entry>schemas</entry>
@@ -4960,6 +4980,314 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
   </table>
  </sect1>
 
+ <sect1 id="catalog-pg-lrg-info">
+  <title><structname>pg_lrg_info</structname></title>
+
+  <indexterm zone="catalog-pg-lrg-info">
+   <primary>pg_lrg_info</primary>
+  </indexterm>
+  <para>
+   The catalog <structname>pg_lrg_info</structname> stores information about
+   node groups. A node group is a group of nodes that send and receive their
+   data changes, and it can be created by the <function>lrg_create</function>
+   function. See <xref linkend="lrg"/> for more information about node groups.
+  </para>
+
+  <para>
+   Each database in the same node group has a tuple with the same information
+   except for <structfield>oid</structfield>.
+  </para>
+
+  <para>
+   Currently, a node can participate in only one node group. This means that
+   this system catalog has at most one tuple.
+  </para>
+
+  <table>
+   <title><structname>pg_lrg_info</structname> Columns</title>
+   <tgroup cols="1">
+    <thead>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       Column Type
+      </para>
+      <para>
+       Description
+      </para></entry>
+     </row>
+    </thead>
+
+    <tbody>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>oid</structfield> <type>oid</type>
+      </para>
+      <para>
+       Row identifier
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>groupname</structfield> <type>name</type>
+      </para>
+      <para>
+       Name of the group
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>puballtables</structfield> <type>bool</type>
+      </para>
+      <para>
+       The type of publication
+      </para></entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+ </sect1>
+
+ <sect1 id="catalog-pg-lrg-nodes">
+  <title><structname>pg_lrg_nodes</structname></title>
+
+  <indexterm zone="catalog-pg-lrg-nodes">
+   <primary>pg_lrg_nodes</primary>
+  </indexterm>
+  <para>
+   The catalog <structname>pg_lrg_nodes</structname> stores information about
+   nodes in node group that this node has attached. When <function>lrg_node_attach</function>()
+   is executed, information about other nodes in a group is copied from the
+   upstream node, and information about this node is copied to the other nodes.
+   Therefore, each database in the same node group has tuples with the same
+   information except for <structfield>oid</structfield> and <structfield>dbid</structfield>.
+   See <xref linkend="lrg"/> for more information about LRG.
+  </para>
+
+  <table>
+   <title><structname>pg_lrg_nodes</structname> Columns</title>
+   <tgroup cols="1">
+    <thead>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       Column Type
+      </para>
+      <para>
+       Description
+      </para></entry>
+     </row>
+    </thead>
+
+    <tbody>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>oid</structfield> <type>oid</type>
+      </para>
+      <para>
+       Row identifier
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>nodeid</structfield> <type>name</type>
+      </para>
+      <para>
+       Identifier of this node
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>groupid</structfield> <type>oid</type>
+       (references <link linkend="catalog-pg-lrg-info"><structname>pg_lrg_info</structname></link>.<structfield>oid</structfield>)
+      </para>
+      <para>
+       Attached group
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>dbid</structfield> <type>oid</type>
+       (references <link linkend="catalog-pg-database"><structname>pg_database</structname></link>.<structfield>oid</structfield>)
+      </para>
+      <para>
+       Database
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>status</structfield> <type>int</type>
+      </para>
+      <para>
+       State code:
+       <literal>0</literal> = initialize,
+       <literal>1</literal> = a publication was created in this node
+       <literal>2</literal> = subscriptions were created in all nodes
+       <literal>3</literal> = ready
+       <literal>4</literal> = this node is now detaching from a group
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>nodename</structfield> <type>name</type>
+      </para>
+      <para>
+       Name of this node
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>localconn</structfield> <type>name</type>
+      </para>
+      <para>
+       Connection string for this node
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>upstreamconn</structfield> <type>name</type>
+      </para>
+      <para>
+       Connection string for upstream node
+      </para></entry>
+     </row>
+
+    </tbody>
+   </tgroup>
+  </table>
+ </sect1>
+
+ <sect1 id="catalog-pg-lrg-pub">
+  <title><structname>pg_lrg_pub</structname></title>
+
+  <indexterm zone="catalog-pg-lrg-pub">
+   <primary>pg_lrg_pub</primary>
+  </indexterm>
+  <para>
+   The catalog <structname>pg_lrg_pub</structname> contains the mapping between
+   node groups and a publication in the database. See <xref linkend="lrg"/> for
+   more information about LRG.
+  </para>
+
+  <table>
+   <title><structname>pg_lrg_pub</structname> Columns</title>
+   <tgroup cols="1">
+    <thead>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       Column Type
+      </para>
+      <para>
+       Description
+      </para></entry>
+     </row>
+    </thead>
+
+    <tbody>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>oid</structfield> <type>oid</type>
+      </para>
+      <para>
+       Row identifier
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>groupid</structfield> <type>oid</type>
+       (references <link linkend="catalog-pg-lrg-info"><structname>pg_lrg_info</structname></link>.<structfield>oid</structfield>)
+      </para>
+      <para>
+       Attached group
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>pubid</structfield> <type>oid</type>
+       (references <link linkend="catalog-pg-publication"><structname>pg_publication</structname></link>.<structfield>oid</structfield>)
+      </para>
+      <para>
+       Publication
+      </para></entry>
+     </row>
+
+
+    </tbody>
+   </tgroup>
+  </table>
+ </sect1>
+
+
+ <sect1 id="catalog-pg-lrg-sub">
+  <title><structname>pg_lrg_sub</structname></title>
+
+  <indexterm zone="catalog-pg-lrg-sub">
+   <primary>pg_lrg_sub</primary>
+  </indexterm>
+  <para>
+   The catalog <structname>pg_lrg_sub</structname> contains the mapping between
+   node groups and subscriptions in the database. See <xref linkend="lrg"/> for
+   more information about LRG.
+  </para>
+
+  <table>
+   <title><structname>pg_lrg_sub</structname> Columns</title>
+   <tgroup cols="1">
+    <thead>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       Column Type
+      </para>
+      <para>
+       Description
+      </para></entry>
+     </row>
+    </thead>
+
+    <tbody>
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>oid</structfield> <type>oid</type>
+      </para>
+      <para>
+       Row identifier
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>groupid</structfield> <type>oid</type>
+       (references <link linkend="catalog-pg-lrg-info"><structname>pg_lrg_info</structname></link>.<structfield>oid</structfield>)
+      </para>
+      <para>
+       Attached group
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>subid</structfield> <type>oid</type>
+       (references <link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link>.<structfield>oid</structfield>)
+      </para>
+      <para>
+       Subscription
+      </para></entry>
+     </row>
+
+
+    </tbody>
+   </tgroup>
+  </table>
+ </sect1>
 
  <sect1 id="catalog-pg-namespace">
   <title><structname>pg_namespace</structname></title>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 40ef5f7ffc..8be17a652e 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -49,6 +49,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY lrg    SYSTEM "lrg.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 85ecc639fd..0722ff8989 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -29233,6 +29233,119 @@ postgres=# SELECT * FROM pg_walfile_name_offset((pg_backup_stop()).lsn);
 
   </sect2>
 
+  <sect2 id="functions-lrg">
+   <title>Logical Replication Group Management Functions</title>
+
+   <para>
+    The functions shown
+    in <xref linkend="functions-lrg-table"/> are for
+    controlling and interacting with node groups.
+   </para>
+
+   <table id="functions-lrg-table">
+    <title>LRG Management Functions</title>
+    <tgroup cols="1">
+     <thead>
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        Function
+       </para>
+       <para>
+        Description
+       </para></entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        <indexterm>
+         <primary></primary>
+        </indexterm>
+        <function>lrg_create</function> ( <parameter>group_name</parameter> <type>name</type>, <parameter>publication_type</parameter> <type>text</type>, <parameter>local_connstring</parameter> <type>text</type>, <parameter>node_name</parameter> <type>name</type> )
+        <returnvalue>void</returnvalue>
+       </para>
+       <para>
+        Creates a node group named <parameter>group_name</parameter>
+        that shares <parameter>publication_type</parameter>, and attach to it as
+        <parameter>node_name</parameter>. If this node is already in a logical
+        replication group, an error is raised. Note that this function returns
+        immediately. This function is just a trigger for starting complex
+        operations by the LRG worker process. If users want to wait until
+        finishing them, <function>lrg_wait</function> can be used.
+       </para></entry>
+      </row>
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        <indexterm>
+         <primary></primary>
+        </indexterm>
+        <function>lrg_node_attach</function> ( <parameter>group_name</parameter> <type>name</type>, <parameter>local_connstring</parameter> <type>text</type>, <parameter>upstream_connstring</parameter> <type>text</type>, <parameter>node_name</parameter> <type>name</type> )
+        <returnvalue>void</returnvalue>
+       </para>
+       <para>
+        Attach to a node group named <parameter>group_name</parameter>
+        as <parameter>node_name</parameter>. The initial data and configuration
+        information of this group will be shared from the <parameter>upstream_connstring</parameter>.
+        <parameter>node_name</parameter> must be unique in the group.
+        If this node is already in a node group, an error is
+        raised. Note that this function returns immediately. This function is
+        just a trigger for starting complex operations by the LRG worker
+        process. If users want to wait until finishing them, <function>lrg_wait</function>
+        can be used.
+       </para></entry>
+      </row>
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        <indexterm>
+         <primary></primary>
+        </indexterm>
+        <function>lrg_node_detach</function> ( <parameter>group_name</parameter> <type>name</type>, <parameter>node_name</parameter> <type>name</type> )
+        <returnvalue>void</returnvalue>
+       </para>
+       <para>
+        Detach <parameter>node_name</parameter> from <parameter>group_name</parameter>.
+        If this node is not in a node group, an error is raised.
+        Note that this function returns immediately. This function is just a
+        trigger for starting complex operations by the LRG worker process. If
+        users want to wait until finishing them, <function>lrg_wait</function>
+        can be used.
+       </para></entry>
+      </row>
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        <indexterm>
+         <primary></primary>
+        </indexterm>
+        <function>lrg_drop</function> ( <parameter>group_name</parameter> <type>name</type> )
+        <returnvalue>void</returnvalue>
+       </para>
+       <para>
+        Detach this node <parameter>group_name</parameter> and remove it. If
+        this node is not in a node group, or there are still nodes
+        in the group, an error is raised.
+       </para></entry>
+      </row>
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        <indexterm>
+         <primary></primary>
+        </indexterm>
+        <function>lrg_wait</function> ()
+        <returnvalue>void</returnvalue>
+       </para>
+       <para>
+        Wait until LRG related operations are done. This function returns
+        immediately if this node is not in a node group.
+       </para></entry>
+      </row>
+
+     </tbody>
+    </tgroup>
+   </table>
+
+  </sect2>
+
   <sect2 id="functions-admin-dbobject">
    <title>Database Object Management Functions</title>
 
diff --git a/doc/src/sgml/lrg.sgml b/doc/src/sgml/lrg.sgml
new file mode 100644
index 0000000000..34843a3ef1
--- /dev/null
+++ b/doc/src/sgml/lrg.sgml
@@ -0,0 +1,417 @@
+<!-- doc/src/sgml/lrg.sgml -->
+ <chapter id="lrg">
+  <title>Logical Replication Group (LRG)</title>
+
+  <indexterm zone="lrg">
+   <primary>logical replication group</primary>
+  </indexterm>
+
+ <para>
+  Logical Replication Group (LRG) is a way to create a node group that
+  replicates data objects and their changes to each other. All nodes in
+  the group can execute Read-Write queries, and its changes will
+  "eventually" send to other nodes.
+ </para>
+
+ <para>
+  When a node is attached to a node group, data that nodes in the group have
+  will be copied to the attaching node. Also, data existing in the node will
+  be copied to others.
+ </para>
+
+ <para>
+  Each node in a node group has one publication and same number of subscriptions
+  as other nodes.
+ </para>
+
+ <para>
+  LRG is especially useful when working with "data-aware" applications.
+  It is recommended that users build a node group located acros
+  different geographical location, and each node inserts or updates different
+  tuples.
+ </para>
+
+ <para>
+  Advantages of LRG are:
+
+  <itemizedlist>
+   <listitem>
+    <para>
+     Allowing load balancing
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     Allowing rolling updates of nodes
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     Improving the availability of the system
+    </para>
+   </listitem>
+  </itemizedlist>
+ </para>
+
+ <sect1 id="lrg-group">
+  <title>Node group</title>
+  <para>
+   Node group is a group that participant nodes have same tables and
+   send/receive their data changes. This is done via <link linkend="logical-replication">logical replication</link>.
+   Each node in a group must be connected to all other nodes. LRG make such a
+   group by creating publications and subscriptions on nodes.
+  </para>
+
+  <para>
+   Node groups are created by <function>lrg_create</function>.
+   The node that executed this function automatically attaches to the created
+   group. This function requires a connection string that is used for
+   connecting to the node from other or connecting to itself. Therefore, it is
+   not recommended to specify the Unix-domain socket or loopback address as the
+   connection string unless testing purposes.
+  </para>
+
+  <para>
+   When users add a node to a node group, the node name must be specified. This
+   can be used for managing node information, and it must be specified when
+   users want to detach a node from a node group. The name must be unique within
+   a node group.
+  </para>
+
+  <para>
+   In order to attach to a node group, the function <function>lrg_node_attach</function>
+   is used. At that time, in addition to specifying connection string for this
+   node, a connection string for connecting to one of the node in the group must
+   be specified. The specified node is called as "upstream node". After
+   executing <function>lrg_node_attach</function>, this node connects to the
+   upstream node, and gets connection strings for other nodes and synchronizes
+   the initial data.
+  </para>
+
+  <para>
+   All tables present in the node group must be set up to allow all changes
+   to be replicated. This requires either of a primary key, or a replica
+   identity with unique index and not-null constraint. LRG does not check them
+   or throw any warnings. Users must define tables properly. Moreover, users
+   must not change any configurations of the automatically defined publication.
+  </para>
+
+  <para>
+   Currently, each node can attach to only one node group, and changes to all
+   tables are replicated to nodes in the group. This is implemented by
+   defining a publication with target list 'FOR ALL TABLES'. This is an
+   implementation restriction that might be lifted in a future release.
+  </para>
+ </sect1>
+
+ <sect1 id="lrg-interface">
+  <title>LRG SQL interface</title>
+  <para>
+   See <xref linkend="functions-lrg"/> for detailed documentation on
+   SQL-level APIs for interacting with LRG.
+  </para>
+ </sect1>
+
+ <sect1 id="lrg-locking">
+  <title>Locking over nodes</title>
+  <para>
+   This feature is under development.
+  </para>
+ </sect1>
+
+
+ <sect1 id="lrg-restriction">
+  <title>Restrictions</title>
+  <para>
+   LRG currently has the following restrictions or missing functionality.
+   <itemizedlist>
+    <listitem>
+     <para>
+      Each node can attach only one node set at a time.
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      All tables in the database must be replicated. That is, only
+      'FOR ALL TABLES' can be specified for <parameter>publication_type</parameter>
+      in <function>lrg_create</function>.
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      All replicated objects must be defined on the all nodes.
+     </para>
+    </listitem>
+    <listitem>
+     <para>
+      LRG does not provide any mechanism for conflict handling (e.g. PK violations).
+      Avoiding conflicts is the user responsibility.
+     </para>
+    </listitem>
+   </itemizedlist>
+  </para>
+ </sect1>
+
+ <sect1 id="lrg-configuration">
+  <title>Configuration</title>
+  <para>
+   LRG will create a publication and subscriptions on each node,
+   that number of subscribers is the number of participants minus one. Therefore LRG requires
+   several configuration options.
+  </para>
+
+  <itemizedlist>
+   <listitem>
+    <para>
+     <varname>wal_level</varname> must be <literal>logical</literal>.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     <varname>max_replication_slots</varname> must be set to at least the number of participants.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     <varname>max_wal_senders</varname> must be set to at least same as <varname>max_replication_slots</varname>.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     <varname>max_logical_replication_workers</varname> must be larger than the number of participants.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     <varname>max_worker_processes</varname> must be larger that <literal>max_logical_replication_workers</literal>.
+    </para>
+   </listitem>
+  </itemizedlist>
+ </sect1>
+
+ <sect1 id="lrg-architecture">
+  <title>Architecture</title>
+
+  <para>
+   LRG is implemented by "LRG launcher" and "LRG worker" processes. The launcher
+   process is registered when the server is not in the recovery mode, and it has
+   a responsibility for starting LRG worker processes. The actual operations for
+   creating or attaching to a node group are done by the LRG
+   worker process. There can be one LRG worker per database.
+  </para>
+
+  <para>
+   More specifically: when SQL functions related with LRG are called by users,
+   tuples will be inserted to catalogs and backend process will notify to the
+   launcher. After receiving the notification, the launcher seek <xref linkend="catalog-pg-lrg-nodes"/>
+   and launches the LRG worker process. This seeking and launching will be
+   also done when server boots.
+  </para>
+ </sect1>
+
+  <sect1 id="lrg-example">
+   <title>Example</title>
+   <para>
+    The following example demonstrates constructing a node group.
+    Assuming that there are three nodes, called Node1, Node2, Node3, and they can connect each other.
+    In the example all nodes are in the same server.
+   </para>
+
+   <sect2>
+    <title>Create a group</title>
+
+    <para>
+     At first, a node group must be created. Here it will be created on Node1.
+    </para>
+<programlisting>
+postgres=# -- Create a node group 'testgroup', and attach to it as 'testnode1'
+postgres=# SELECT lrg_create('testgroup', 'FOR ALL TABLES', 'port=5431 dbname=postgres', 'testnode1');
+ lrg_create
+------------
+
+(1 row)
+
+postgres=# -- Wait until operations by LRG worker are done
+postgres=# SELECT lrg_wait();
+ lrg_wait
+----------
+
+(1 row)
+
+</programlisting>
+
+    <para>
+     Information of the group and nodes can check via system catalogs,
+     like <link linkend="catalog-pg-lrg-nodes"><structname>pg_lrg_nodes</structname></link>.
+    </para>
+<programlisting>
+postgres=# SELECT * FROM pg_lrg_info ;
+  oid  | groupname | puballtables
+-------+-----------+--------------
+ 16384 | testgroup | t
+(1 row)
+
+postgres=# SELECT * FROM pg_lrg_nodes;
+  oid  |        nodeid        | groupid | dbid | status | nodename  |         localconn         | upstreamconn
+-------+----------------------+---------+------+--------+-----------+---------------------------+--------------
+ 16385 | 70988980716274892555 |   16384 |    5 |      3 | testnode1 | port=5431 dbname=postgres |
+(1 row)
+</programlisting>
+
+   </sect2>
+
+   <sect2>
+    <title>Attach to a group</title>
+
+    <para>
+     Next Node2 can attach the created group. Note again that the connection string for Node1 must be specified.
+    </para>
+
+<programlisting>
+postgres=# -- Attach to 'testgroup' as 'testnode2'
+postgres=# SELECT lrg_node_attach('testgroup', 'port=5432 dbname=postgres', 'port=5431 dbname=postgres', 'testnode2');
+ lrg_node_attach
+-----------------
+
+(1 row)
+
+postgres=# -- Wait until operations by LRG worker are done
+postgres=# SELECT lrg_wait();
+ lrg_wait
+----------
+
+(1 row)
+</programlisting>
+
+    <para>
+     The status of all nodes can check via pg_lrg_nodes. Following tuples will be found on both nodes.
+    </para>
+
+<programlisting>
+postgres=# SELECT * FROM pg_lrg_info ;
+  oid  | groupname | puballtables
+-------+-----------+--------------
+ 16384 | testgroup | t
+(1 row)
+
+postgres=# SELECT * FROM pg_lrg_nodes;
+  oid  |        nodeid        | groupid | dbid | status | nodename  |         localconn         |       upstreamconn
+-------+----------------------+---------+------+--------+-----------+---------------------------+---------------------------
+ 16385 | 70989229890284027935 |   16384 |    5 |      3 | testnode2 | port=5432 dbname=postgres | port=5431 dbname=postgres
+ 16386 | 70988980716274892555 |   16385 |    5 |      3 | testnode1 | port=5431 dbname=postgres |
+(2 rows)
+</programlisting>
+
+    <para>
+     Now Node1 and Node2 has same contents about LRG system catalogs,
+     so ether of them can be specified as an upstream node.
+     In below example Node2 is used as upstream for attaching a new node.
+    </para>
+
+<programlisting>
+postgres=# -- Attach to 'testgroup' as 'testode3', and data will be copied from Node2
+postgres=# SELECT lrg_node_attach('testgroup', 'port=5433 dbname=postgres', 'port=5432 dbname=postgres', 'testnode3');
+ lrg_node_attach
+-----------------
+
+(1 row)
+
+postgres=# -- Wait until operations by LRG worker are done
+postgres=# SELECT lrg_wait();
+ lrg_wait
+----------
+
+(1 row)
+</programlisting>
+
+    <para>
+     Finally pg_lrg_info and pg_lrg_nodes will be like:
+    </para>
+
+<programlisting>
+postgres=# SELECT * FROM pg_lrg_info ;
+  oid  | groupname | puballtables
+-------+-----------+--------------
+ 16384 | testgroup | t
+(1 row)
+
+postgres=# SELECT * FROM pg_lrg_nodes;
+  oid  |        nodeid        | groupid | dbid | status | nodename  |         localconn         |       upstreamconn
+-------+----------------------+---------+------+--------+-----------+---------------------------+---------------------------
+ 16385 | 70989243367269230745 |   16384 |    5 |      3 | testnode3 | port=5433 dbname=postgres | port=5432 dbname=postgres
+ 16386 | 70989229890284027935 |   16385 |    5 |      3 | testnode2 | port=5432 dbname=postgres | port=5431 dbname=postgres
+ 16387 | 70988980716274892555 |   16385 |    5 |      3 | testnode1 | port=5431 dbname=postgres |
+(3 rows)
+</programlisting>
+
+    <para>
+     Now all nodes publish their changes, and they subscribe them. If a tuple inserted on Node1,
+     data will be also found on Node2 and Node3.
+    </para>
+
+   </sect2>
+
+   <sect2>
+    <title>Detach from a group</title>
+    <para>
+     User can detach attached nodes by executing <function>lrg_node_detach</function>
+     at any time. This function must be called from a node that is a member of a
+     group.
+    </para>
+
+<programlisting>
+postgres=# -- Detach Node3 from 'testgroup'. This can be done from the arbitrary member of the group
+postgres=# select lrg_node_detach('testgroup', 'testnode3');
+ lrg_node_detach
+-----------------
+
+(1 row)
+
+postgres=# -- Also detach Node2 from 'testgroup'. This can be done from Node1 or Node2
+postgres=# select lrg_node_detach('testgroup', 'testnode2');
+ lrg_node_detach
+-----------------
+
+(1 row)
+</programlisting>
+
+    <para>
+     The user-defined data will be not removed even if nodes are detached, but tuples in LRG related catalogs will be deleted.
+    </para>
+
+<programlisting>
+postgres=# -- On Node2, there are no tuples in pg_lrg_nodes
+postgres=# select * from pg_lrg_nodes;
+ oid | nodeid | groupid | dbid | status | nodename | localconn | upstreamconn
+-----+--------+---------+------+--------+----------+-----------+--------------
+(0 rows)
+</programlisting>
+   </sect2>
+
+   <sect2>
+    <title>Drop a group</title>
+    <para>
+     For dropping a group, the API <function>lrg_drop</function> can be used.
+     The function must be callled from a node that is a member of a group, and
+     it will throw ERROR if there are other members in the group.
+    </para>
+
+<programlisting>
+postgres=# -- Drop 'testgroup.' Node1 will detach from it automatically.
+postgres=# select lrg_drop('testgroup');
+ lrg_drop
+----------
+
+(1 row)
+
+postgres=# -- There are no tuples in pg_lrg_info
+postgres=# select * from pg_lrg_info;
+ oid | groupname | puballtables
+-----+-----------+--------------
+(0 rows)
+</programlisting>
+   </sect2>
+
+  </sect1>
+
+ </chapter>
\ No newline at end of file
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index 0b60e46d69..bcea47fdc9 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -172,6 +172,7 @@ break is not needed in a wider output rendering.
   &logical-replication;
   &jit;
   &regress;
+  &lrg;
 
  </part>
 
-- 
2.27.0

