From f9066c1497ab22312cd28dc3d204ca3b5220350c Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathandbossart@gmail.com>
Date: Fri, 9 Dec 2022 19:40:54 -0800
Subject: [PATCH v1 3/3] Allow recovery via loadable modules.

This adds the restore_library, archive_cleanup_library, and
recovery_end_library parameters to allow archive recovery via a
loadable module, rather than running shell commands.
---
 contrib/basic_archive/Makefile                |   4 +-
 contrib/basic_archive/basic_archive.c         |  69 ++++++-
 contrib/basic_archive/meson.build             |   7 +-
 contrib/basic_archive/t/001_restore.pl        |  42 +++++
 doc/src/sgml/archive-modules.sgml             | 176 ++++++++++++++++--
 doc/src/sgml/backup.sgml                      |  40 +++-
 doc/src/sgml/basic-archive.sgml               |  33 ++--
 doc/src/sgml/config.sgml                      |  99 +++++++++-
 doc/src/sgml/high-availability.sgml           |  18 +-
 src/backend/access/transam/shell_restore.c    |  21 ++-
 src/backend/access/transam/xlog.c             |  13 +-
 src/backend/access/transam/xlogarchive.c      | 131 ++++++++++++-
 src/backend/access/transam/xlogrecovery.c     |  20 +-
 src/backend/postmaster/checkpointer.c         |  23 +++
 src/backend/postmaster/startup.c              |  22 ++-
 src/backend/utils/misc/guc_tables.c           |  30 +++
 src/backend/utils/misc/postgresql.conf.sample |   3 +
 src/include/access/xlog_internal.h            |   1 +
 src/include/access/xlogarchive.h              |  43 ++++-
 src/include/access/xlogrecovery.h             |   3 +
 20 files changed, 724 insertions(+), 74 deletions(-)
 create mode 100644 contrib/basic_archive/t/001_restore.pl

diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 55d299d650..487dc563f3 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -1,7 +1,7 @@
 # contrib/basic_archive/Makefile
 
 MODULES = basic_archive
-PGFILEDESC = "basic_archive - basic archive module"
+PGFILEDESC = "basic_archive - basic archive and recovery module"
 
 REGRESS = basic_archive
 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
@@ -9,6 +9,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
 # which typical installcheck users do not have (e.g. buildfarm clients).
 NO_INSTALLCHECK = 1
 
+TAP_TESTS = 1
+
 ifdef USE_PGXS
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 9f221816bb..2a702455ca 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
  * a file is successfully archived and then the system crashes before
  * a durable record of the success has been made.
  *
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ *		cp /path/to/archivedir/%f %p
+ *
  * Copyright (c) 2022, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
@@ -30,6 +35,7 @@
 #include <sys/time.h>
 #include <unistd.h>
 
+#include "access/xlogarchive.h"
 #include "common/int.h"
 #include "miscadmin.h"
 #include "postmaster/pgarch.h"
@@ -48,6 +54,8 @@ static bool basic_archive_file(const char *file, const char *path);
 static void basic_archive_file_internal(const char *file, const char *path);
 static bool check_archive_directory(char **newval, void **extra, GucSource source);
 static bool compare_files(const char *file1, const char *file2);
+static bool basic_restore_file(const char *file, const char *path,
+							   const char *lastRestartPointFileName);
 
 /*
  * _PG_init
@@ -87,6 +95,19 @@ _PG_archive_module_init(ArchiveModuleCallbacks *cb)
 	cb->archive_file_cb = basic_archive_file;
 }
 
+/*
+ * _PG_recovery_module_init
+ *
+ * Returns the module's restore callback.
+ */
+void
+_PG_recovery_module_init(RecoveryModuleCallbacks *cb)
+{
+	AssertVariableIsOfType(&_PG_recovery_module_init, RecoveryModuleInit);
+
+	cb->restore_cb = basic_restore_file;
+}
+
 /*
  * check_archive_directory
  *
@@ -99,8 +120,8 @@ check_archive_directory(char **newval, void **extra, GucSource source)
 
 	/*
 	 * The default value is an empty string, so we have to accept that value.
-	 * Our check_configured callback also checks for this and prevents
-	 * archiving from proceeding if it is still empty.
+	 * Our check_configured and restore callbacks also check for this and
+	 * prevent archiving or recovery from proceeding if it is still empty.
 	 */
 	if (*newval == NULL || *newval[0] == '\0')
 		return true;
@@ -368,3 +389,47 @@ compare_files(const char *file1, const char *file2)
 
 	return ret;
 }
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file from the WAL archives.
+ */
+static bool
+basic_restore_file(const char *file, const char *path,
+				   const char *lastRestartPointFileName)
+{
+	char		source[MAXPGPATH];
+	struct stat st;
+
+	ereport(DEBUG1,
+			(errmsg("restoring \"%s\" via basic_archive", file)));
+
+	if (archive_directory == NULL || archive_directory[0] == '\0')
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("\"basic_archive.archive_directory\" is not set")));
+
+	/*
+	 * Check whether the file exists.  If not, we return false to indicate that
+	 * there are no more files to restore.
+	 */
+	snprintf(source, MAXPGPATH, "%s/%s", archive_directory, file);
+	if (stat(source, &st) != 0)
+	{
+		int		elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not stat file \"%s\": %m", source)));
+		return false;
+	}
+
+	copy_file(source, unconstify(char *, path));
+	fsync_fname(path, false);
+	fsync_fname(archive_directory, true);
+
+	ereport(DEBUG1,
+			(errmsg("restored \"%s\" via basic_archive", file)));
+	return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index b25dce99a3..5b5a6d79e7 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -7,7 +7,7 @@ basic_archive_sources = files(
 if host_system == 'windows'
   basic_archive_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
     '--NAME', 'basic_archive',
-    '--FILEDESC', 'basic_archive - basic archive module',])
+    '--FILEDESC', 'basic_archive - basic archive and recovery module',])
 endif
 
 basic_archive = shared_module('basic_archive',
@@ -31,4 +31,9 @@ tests += {
     # which typical runningcheck users do not have (e.g. buildfarm clients).
     'runningcheck': false,
   },
+  'tap': {
+    'tests': [
+      't/001_restore.pl',
+    ],
+  },
 }
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..86ff0dc7f5
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,42 @@
+
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL was replayed
+my $result = $restore->safe_psql("postgres", "SELECT count(*) FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-modules.sgml b/doc/src/sgml/archive-modules.sgml
index ef02051f7f..023f2e047a 100644
--- a/doc/src/sgml/archive-modules.sgml
+++ b/doc/src/sgml/archive-modules.sgml
@@ -1,34 +1,43 @@
 <!-- doc/src/sgml/archive-modules.sgml -->
 
 <chapter id="archive-modules">
- <title>Archive Modules</title>
+ <title>Archive and Recovery Modules</title>
  <indexterm zone="archive-modules">
-  <primary>Archive Modules</primary>
+  <primary>Archive and Recovery Modules</primary>
  </indexterm>
 
  <para>
   PostgreSQL provides infrastructure to create custom modules for continuous
-  archiving (see <xref linkend="continuous-archiving"/>).  While archiving via
-  a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
-  simpler, a custom archive module will often be considerably more robust and
-  performant.
+  archiving and recovery (see <xref linkend="continuous-archiving"/>).  While
+  a shell command (e.g., <xref linkend="guc-archive-command"/>,
+  <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+  often be considerably more robust and performant.
  </para>
 
  <para>
   When a custom <xref linkend="guc-archive-library"/> is configured, PostgreSQL
   will submit completed WAL files to the module, and the server will avoid
   recycling or removing these WAL files until the module indicates that the files
-  were successfully archived.  It is ultimately up to the module to decide what
-  to do with each WAL file, but many recommendations are listed at
-  <xref linkend="backup-archiving-wal"/>.
+  were successfully archived.  When a custom
+  <xref linkend="guc-restore-library"/>,
+  <xref linkend="guc-archive-cleanup-library"/>, or
+  <xref linkend="guc-recovery-end-library"/> is configured, PostgreSQL will use
+  the module for the corresponding recovery action.  It is ultimately up to the
+  module to decide how to accomplish each task, but some recommendations are
+  listed at <xref linkend="backup-archiving-wal"/> and
+  <xref linkend="backup-pitr-recovery"/>.
  </para>
 
  <para>
-  Archiving modules must at least consist of an initialization function (see
-  <xref linkend="archive-module-init"/>) and the required callbacks (see
-  <xref linkend="archive-module-callbacks"/>).  However, archive modules are
-  also permitted to do much more (e.g., declare GUCs and register background
-  workers).
+  Archive and recovery modules must at least consist of an initialization
+  function (see <xref linkend="archive-module-init"/> and
+  <xref linkend="recovery-module-init"/>) and the required callbacks (see
+  <xref linkend="archive-module-callbacks"/> and
+  <xref linkend="recovery-module-callbacks"/>).  However, archive and recovery
+  modules are also permitted to do much more (e.g., declare GUCs and register
+  background workers).  A module may be used for any combination of the
+  configuration parameters mentioned in the preceding paragraph, or it can be
+  used for just one.
  </para>
 
  <para>
@@ -37,7 +46,7 @@
  </para>
 
  <sect1 id="archive-module-init">
-  <title>Initialization Functions</title>
+  <title>Archive Module Initialization Functions</title>
   <indexterm zone="archive-module-init">
    <primary>_PG_archive_module_init</primary>
   </indexterm>
@@ -64,6 +73,12 @@ typedef void (*ArchiveModuleInit) (struct ArchiveModuleCallbacks *cb);
    Only the <function>archive_file_cb</function> callback is required.  The
    others are optional.
   </para>
+
+  <note>
+   <para>
+    <varname>archive_library</varname> is only loaded in the archiver process.
+   </para>
+  </note>
  </sect1>
 
  <sect1 id="archive-module-callbacks">
@@ -133,4 +148,135 @@ typedef void (*ArchiveShutdownCB) (void);
    </para>
   </sect2>
  </sect1>
+
+ <sect1 id="recovery-module-init">
+  <title>Recovery Module Initialization Functions</title>
+  <indexterm zone="recovery-module-init">
+   <primary>_PG_recovery_module_init</primary>
+  </indexterm>
+  <para>
+   A recovery library is loaded by dynamically loading a shared library with the
+   <xref linkend="guc-restore-library"/>,
+   <xref linkend="guc-archive-cleanup-library"/>, or
+   <xref linkend="guc-recovery-end-library"/> as the library base name.  The
+   normal library search path is used to locate the library.  To provide the
+   required recovery module callbacks and to indicate that the library is
+   actually a recovery module, it needs to provide a function named
+   <function>_PG_recovery_module_init</function>.  This function is passed a
+   struct that needs to be filled with the callback function pointers for
+   individual actions.
+
+<programlisting>
+typedef struct RecoveryModuleCallbacks
+{
+    RecoveryRestoreCB restore_cb;
+    RecoveryArchiveCleanupCB archive_cleanup_cb;
+    RecoveryEndCB recovery_end_cb;
+} RecoveryModuleCallbacks;
+typedef void (*RecoveryModuleInit) (struct RecoveryModuleCallbacks *cb);
+</programlisting>
+
+   If the recovery module is loaded via <varname>restore_library</varname>, the
+   <function>restore_cb</function> callback is required.  If the recovery
+   module is loaded via <varname>archive_cleanup_library</varname>, the
+   <function>archive_cleanup_cb</function> callback is required.  If the
+   recovery module is loaded via <varname>recovery_end_library</varname>, the
+   <function>recovery_end_library</function> callback is required.  The same
+   recovery module may be used for more than one of the aforementioned
+   parameters if desired.  Unused callback functions (e.g., if
+   <function>restore_cb</function> is defined but the library is only loaded
+   via <varname>recovery_end_library</varname>) are ignored.
+  </para>
+
+  <note>
+   <para>
+    <varname>restore_library</varname> and
+    <varname>recovery_end_library</varname> are only loaded in the startup
+    process and in single-user mode, while
+    <varname>archive_cleanup_library</varname> is only loaded in the
+    checkpointer process.
+   </para>
+  </note>
+
+  <note>
+   <para>
+    A recovery module's <function>_PG_recovery_module_init</function> might be
+    called multiple times in the same process.  If a module uses this function
+    for anything beyond returning its callback functions, it must be able to cope
+    with multiple invocations.
+   </para>
+  </note>
+ </sect1>
+
+ <sect1 id="recovery-module-callbacks">
+  <title>Recovery Module Callbacks</title>
+  <para>
+   The recovery callbacks define the actual behavior of the module.  The server
+   will call them as required to execute recovery actions.
+  </para>
+
+  <sect2 id="recovery-module-restore">
+   <title>Restore Callback</title>
+   <para>
+    The <function>restore_cb</function> callback is called to retrieve a single
+    archived segment of the WAL file series for archive recovery or streaming
+    replication.
+
+<programlisting>
+typedef bool (*RecoveryRestoreCB) (const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+    This callback must return <literal>true</literal> only if the file was
+    successfully retrieved.  If the file is not available in the archives, the
+    callback must return <literal>false</literal>.
+    <replaceable>file</replaceable> will contain just the file name
+    of the WAL file to retrieve, while <replaceable>path</replaceable> contains
+    the destination's relative path (including the file name).
+    <replaceable>lastRestartPointFileName</replaceable> will contain the name
+    of the file containing the last valid restart point.  That is the earliest
+    file that must be kept to allow a restore to be restartable, so this
+    information can be used to truncate the archive to just the minimum
+    required to support restarting from the current restore.
+    <replaceable>lastRestartPointFileName</replaceable> is typically only used
+    by warm-standby configurations (see <xref linkend="warm-standby"/>).  Note
+    that if multiple standby servers are restoring from the same archive
+    directory, you will need to ensure that you do not delete WAL files until
+    they are no longer needed by any of the servers.
+   </para>
+  </sect2>
+
+  <sect2 id="recovery-module-archive-cleanup">
+   <title>Archive Cleanup Callback</title>
+   <para>
+    The <function>archive_cleanup_cb</function> callback is called at every
+    restart point and is intended to provide a mechanism for cleaning up old
+    archived WAL files that are no longer needed by the standby server.
+
+<programlisting>
+typedef void (*RecoveryArchiveCleanupCB) (const char *lastRestartPointFileName);
+</programlisting>
+
+    <replaceable>lastRestartPointFileName</replaceable> will contain the name
+    of the file containing the last valid restart point, like in
+    <link linkend="recovery-module-restore"><function>restore_cb</function></link>.
+   </para>
+  </sect2>
+
+  <sect2 id="recovery-module-end">
+   <title>Recovery End Callback</title>
+   <para>
+    The <function>recovery_end_cb</function> callback is called once at the end
+    of recovery and is intended to provide a mechanism for cleanup following
+    replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (const char *lastRestartPointFileName);
+</programlisting>
+
+    <replaceable>lastRestartPointFileName</replaceable> will contain the name
+    of the file containing the last valid restart point, like in
+    <link linkend="recovery-module-restore"><function>restore_cb</function></link>.
+   </para>
+  </sect2>
+ </sect1>
 </chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 8bab521718..8cf3f35649 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1181,9 +1181,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
    <para>
     The key part of all this is to set up a recovery configuration that
     describes how you want to recover and how far the recovery should
-    run.  The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+    run.  The one thing that you absolutely must specify is either
+    <varname>restore_command</varname> or <varname>restore_library</varname>,
     which tells <productname>PostgreSQL</productname> how to retrieve archived
-    WAL file segments.  Like the <varname>archive_command</varname>, this is
+    WAL file segments.
+   </para>
+
+   <para>
+    Like the <varname>archive_library</varname> parameter,
+    <varname>restore_library</varname> is a shared library.  Since such
+    libraries are written in <literal>C</literal>, creating your own may
+    require considerably more effort than writing a shell command.  However,
+    recovery modules can be more performant than restoring via shell, and they
+    will have access to many useful server resources.  For more information
+    about creating a <varname>restore_library</varname>, see
+    <xref linkend="archive-modules"/>.
+   </para>
+
+   <para>
+    Like the <varname>archive_command</varname>,
+    <varname>restore_command</varname> is
     a shell command string.  It can contain <literal>%f</literal>, which is
     replaced by the name of the desired WAL file, and <literal>%p</literal>,
     which is replaced by the path name to copy the WAL file to.
@@ -1202,14 +1219,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
    </para>
 
    <para>
-    It is important that the command return nonzero exit status on failure.
-    The command <emphasis>will</emphasis> be called requesting files that are not
-    present in the archive; it must return nonzero when so asked.  This is not
-    an error condition.  An exception is that if the command was terminated by
+    It is important that the <varname>restore_command</varname> return nonzero
+    exit status on failure, or, if you are using a
+    <varname>restore_library</varname>, that the restore function returns
+    <literal>false</literal> on failure.  The command or library
+    <emphasis>will</emphasis> be called requesting files that are not
+    present in the archive; it must fail when so asked.  This is not
+    an error condition.  An exception is that if the
+    <varname>restore_command</varname> was terminated by
     a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
     part of a database server shutdown) or an error by the shell (such as
     command not found), then recovery will abort and the server will not start
-    up.
+    up.  Likewise, if the restore function provided by the
+    <varname>restore_library</varname> emits an <literal>ERROR</literal> or
+    <literal>FATAL</literal>, recovery will abort and the server won't start.
    </para>
 
    <para>
@@ -1233,7 +1256,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
     close as possible given the available WAL segments).  Therefore, a normal
     recovery will end with a <quote>file not found</quote> message, the exact text
     of the error message depending upon your choice of
-    <varname>restore_command</varname>.  You may also see an error message
+    <varname>restore_command</varname> or <varname>restore_library</varname>.
+    You may also see an error message
     at the start of recovery for a file named something like
     <filename>00000001.history</filename>.  This is also normal and does not
     indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index 0b650f17a8..ac7cd9b967 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
  </indexterm>
 
  <para>
-  <filename>basic_archive</filename> is an example of an archive module.  This
-  module copies completed WAL segment files to the specified directory.  This
-  may not be especially useful, but it can serve as a starting point for
-  developing your own archive module.  For more information about archive
-  modules, see <xref linkend="archive-modules"/>.
+  <filename>basic_archive</filename> is an example of an archive and recovery
+  module.  This module copies completed WAL segment files to or from the
+  specified directory.  This may not be especially useful, but it can serve as
+  a starting point for developing your own archive and recovery modules.  For
+  more information about archive and recovery modules, see
+  see <xref linkend="archive-modules"/>.
  </para>
 
  <para>
-  In order to function, this module must be loaded via
+  For use as an archive module, this module must be loaded via
   <xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
-  must be enabled.
+  must be enabled.  For use as a recovery module, this module must be loaded
+  via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+  <xref linkend="runtime-config-wal-archive-recovery"/>).
  </para>
 
  <sect2>
@@ -34,11 +37,12 @@
     </term>
     <listitem>
      <para>
-      The directory where the server should copy WAL segment files.  This
-      directory must already exist.  The default is an empty string, which
-      effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
-      is enabled, the server will accumulate WAL segment files in the
-      expectation that a value will soon be provided.
+      The directory where the server should copy WAL segment files to or from.
+      This directory must already exist.  The default is an empty string,
+      which, when used for archiving, effectively halts WAL archival, but if
+      <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+      WAL segment files in the expectation that a value will soon be provided.
+      When an empty string is used for recovery, restore will fail.
      </para>
     </listitem>
    </varlistentry>
@@ -46,7 +50,7 @@
 
   <para>
    These parameters must be set in <filename>postgresql.conf</filename>.
-   Typical usage might be:
+   Typical usage as an archive module might be:
   </para>
 
 <programlisting>
@@ -61,7 +65,8 @@ basic_archive.archive_directory = '/path/to/archive/directory'
   <title>Notes</title>
 
   <para>
-   Server crashes may leave temporary files with the prefix
+   When <filename>basic_archive</filename> is used as an archive module, server
+   crashes may leave temporary files with the prefix
    <filename>archtemp</filename> in the archive directory.  It is recommended to
    delete such files before restarting the server after a crash.  It is safe to
    remove such files while the server is running as long as they are unrelated
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 3071c8eace..5cf9d0a42a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3773,7 +3773,8 @@ include_dir 'conf.d'
      recovery when the end of archived WAL is reached, but will keep trying to
      continue recovery by connecting to the sending server as specified by the
      <varname>primary_conninfo</varname> setting and/or by fetching new WAL
-     segments using <varname>restore_command</varname>.  For this mode, the
+     segments using <varname>restore_command</varname> or
+     <varname>restore_library</varname>.  For this mode, the
      parameters from this section and <xref
      linkend="runtime-config-replication-standby"/> are of interest.
      Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3801,7 +3802,8 @@ include_dir 'conf.d'
       <listitem>
        <para>
         The local shell command to execute to retrieve an archived segment of
-        the WAL file series. This parameter is required for archive recovery,
+        the WAL file series. Either <varname>restore_command</varname> or
+        <xref linkend="guc-restore-library"/> is required for archive recovery,
         but optional for streaming replication.
         Any <literal>%f</literal> in the string is
         replaced by the name of the file to retrieve from the archive,
@@ -3836,7 +3838,36 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
 
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
-        file or on the server command line.
+        file or on the server command line.  It is only used if
+        <varname>restore_library</varname> is set to an empty string.  If both
+        <varname>restore_command</varname> and
+        <varname>restore_library</varname> are set, an error will be raised.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="guc-restore-library" xreflabel="restore_library">
+      <term><varname>restore_library</varname> (<type>string</type>)
+      <indexterm>
+        <primary><varname>restore_library</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        The library to use for retrieving an archived segment of the WAL file
+        series.  Either <xref linkend="guc-restore-command"/> or
+        <varname>restore_library</varname> is required for archive recovery,
+        but optional for streaming replication.  If this parameter is set to an
+        empty string (the default), restoring via shell is enabled, and
+        <varname>restore_command</varname> is used.  If both
+        <varname>restore_command</varname> and
+        <varname>restore_library</varname> are set, an error will be raised.
+        Otherwise, the specified shared library is used for restoring.  For
+        more information, see <xref linkend="archive-modules"/>.
+       </para>
+
+       <para>
+        This parameter can only be set at server start.
        </para>
       </listitem>
      </varlistentry>
@@ -3881,7 +3912,37 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
        </para>
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
-        file or on the server command line.
+        file or on the server command line.  It is only used if
+        <varname>archive_cleanup_library</varname> is set to an empty string.
+        If both <varname>archive_cleanup_command</varname> and
+        <varname>archive_cleanup_library</varname> are set, an error will be
+        raised.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="guc-archive-cleanup-library" xreflabel="archive_cleanup_library">
+      <term><varname>archive_cleanup_library</varname> (<type>string</type>)
+      <indexterm>
+        <primary><varname>archive_cleanup_library</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This optional parameter specifies a library that will be executed at
+        every restartpoint.  The purpose of this parameter is to provide a
+        mechanism for cleaning up old archived WAL files that are no longer
+        needed by the standby server.  If this parameter is set to an empty
+        string (the default), the shell command specified in
+        <xref linkend="guc-archive-cleanup-command"/> is used.  If both
+        <varname>archive_cleanup_command</varname> and
+        <varname>archive_cleanup_library</varname> are set, an error will be
+        raised.  Otherwise, the specified shared library is used for cleanup.
+        For more information, see <xref linkend="archive-modules"/>.
+       </para>
+
+       <para>
+        This parameter can only be set at server start.
        </para>
       </listitem>
      </varlistentry>
@@ -3910,11 +3971,39 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
        </para>
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
-        file or on the server command line.
+        file or on the server command line.  It is only used if
+        <varname>recovery_end_library</varname> is set to an empty string.  If
+        both <varname>recovery_end_command</varname> and
+        <varname>recovery_end_library</varname> are set, an error will be
+        raised.
        </para>
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-recovery-end-library" xreflabel="recovery_end_library">
+      <term><varname>recovery_end_library</varname> (<type>string</type>)
+      <indexterm>
+        <primary><varname>recovery_end_library</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This optional parameter specifies a library that will be executed once
+        only at the end of recovery.  The purpose of this parameter is to
+        provide a mechanism for cleanup following replication or recovery.  If
+        this parameter is set to an empty string (the default), the shell
+        command specified in <xref linkend="guc-recovery-end-command"/> is
+        used.  If both <varname>recovery_end_command</varname> and
+        <varname>recovery_end_library</varname> are set, an error will be
+        raised.  Otherwise, the specified shared library is used for cleanup.
+        For more information, see <xref linkend="archive-modules"/>.
+       </para>
+
+       <para>
+        This parameter can only be set at server start.
+       </para>
+      </listitem>
+     </varlistentry>
     </variablelist>
 
   </sect2>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index f180607528..963d12e02a 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
    <para>
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
-    (see <xref linkend="guc-restore-command"/>) or directly from the primary
+    (see <xref linkend="guc-restore-command"/> and
+    <xref linkend="guc-restore-library"/>) or directly from the primary
     over a TCP connection (streaming replication). The standby server will
     also attempt to restore any WAL found in the standby cluster's
     <filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
 
    <para>
     At startup, the standby begins by restoring all WAL available in the
-    archive location, calling <varname>restore_command</varname>. Once it
+    archive location, either by calling <varname>restore_command</varname> or
+    by executing the <varname>restore_library</varname>. Once it
     reaches the end of WAL available there and <varname>restore_command</varname>
+    or <varname>restore_library</varname>
     fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
     If that fails, and streaming replication has been configured, the
     standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
     server (see <xref linkend="backup-pitr-recovery"/>). Create a file
     <link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
     in the standby's cluster data
-    directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+    directory. Set <xref linkend="guc-restore-command"/> or
+    <xref linkend="guc-restore-library"/> to copy files from
     the WAL archive. If you plan to have multiple standby servers for high
     availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
     <literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
 
    <note>
      <para>
-     <xref linkend="guc-restore-command"/> should return immediately
+     <xref linkend="guc-restore-command"/> and
+     <xref linkend="guc-restore-library"/> should return immediately
      if the file does not exist; the server will retry the command again if
      necessary.
     </para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
 
    <para>
     If you're using a WAL archive, its size can be minimized using the <xref
-    linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
-    longer required by the standby server.
+    linkend="guc-archive-cleanup-command"/> or
+    <xref linkend="guc-archive-cleanup-library"/> parameter to remove files
+    that are no longer required by the standby server.
     The <application>pg_archivecleanup</application> utility is designed specifically to
     be used with <varname>archive_cleanup_command</varname> in typical single-standby
     configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/shell_restore.c b/src/backend/access/transam/shell_restore.c
index 073e709e06..124a0bbdb6 100644
--- a/src/backend/access/transam/shell_restore.c
+++ b/src/backend/access/transam/shell_restore.c
@@ -3,7 +3,8 @@
  * shell_restore.c
  *
  * These recovery functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC).  It is used as the default, but other modules may
+ * define their own recovery logic.
  *
  * Copyright (c) 2022, PostgreSQL Global Development Group
  *
@@ -22,6 +23,10 @@
 #include "storage/ipc.h"
 #include "utils/wait_event.h"
 
+static bool shell_restore(const char *file, const char *path,
+						  const char *lastRestartPointFileName);
+static void shell_archive_cleanup(const char *lastRestartPointFileName);
+static void shell_recovery_end(const char *lastRestartPointFileName);
 static char *BuildCleanupCommand(const char *command,
 								 const char *lastRestartPointFileName);
 static bool ExecuteRecoveryCommand(const char *command,
@@ -29,6 +34,16 @@ static bool ExecuteRecoveryCommand(const char *command,
 								   bool exitOnSigterm, uint32 wait_event_info,
 								   int fail_elevel);
 
+void
+shell_restore_init(RecoveryModuleCallbacks *cb)
+{
+	AssertVariableIsOfType(&shell_restore_init, RecoveryModuleInit);
+
+	cb->restore_cb = shell_restore;
+	cb->archive_cleanup_cb = shell_archive_cleanup;
+	cb->recovery_end_cb = shell_recovery_end;
+}
+
 bool
 shell_restore(const char *file, const char *path,
 			  const char *lastRestartPointFileName)
@@ -73,7 +88,7 @@ shell_restore(const char *file, const char *path,
 	return ret;
 }
 
-void
+static void
 shell_archive_cleanup(const char *lastRestartPointFileName)
 {
 	char	   *cmd = BuildCleanupCommand(archiveCleanupCommand,
@@ -84,7 +99,7 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
 	pfree(cmd);
 }
 
-void
+static void
 shell_recovery_end(const char *lastRestartPointFileName)
 {
 	char	   *cmd = BuildCleanupCommand(recoveryEndCommand,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 32225be4a5..6365635180 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -4883,10 +4883,11 @@ static void
 CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
 							TimeLineID newTLI)
 {
+
 	/*
-	 * Execute the recovery_end_command, if any.
+	 * Execute the recovery-end library, if any.
 	 */
-	if (recoveryEndCommand && strcmp(recoveryEndCommand, "") != 0)
+	if (recoveryEndCommand[0] != '\0' || recoveryEndLibrary[0] != '\0')
 	{
 		char		lastRestartPointFname[MAXPGPATH];
 		XLogSegNo	restartSegNo;
@@ -4903,7 +4904,7 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
 		XLogFileName(lastRestartPointFname, restartTli, restartSegNo,
 					 wal_segment_size);
 
-		shell_recovery_end(lastRestartPointFname);
+		RecoveryContext.recovery_end_cb(lastRestartPointFname);
 	}
 
 	/*
@@ -7318,9 +7319,9 @@ CreateRestartPoint(int flags)
 							   timestamptz_to_str(xtime)) : 0));
 
 	/*
-	 * Finally, execute archive_cleanup_command, if any.
+	 * Execute the archive-cleanup library, if any.
 	 */
-	if (archiveCleanupCommand && strcmp(archiveCleanupCommand, "") != 0)
+	if (archiveCleanupCommand[0] != '\0' || archiveCleanupLibrary[0] != '\0')
 	{
 		char		lastRestartPointFname[MAXPGPATH];
 		XLogSegNo	restartSegNo;
@@ -7337,7 +7338,7 @@ CreateRestartPoint(int flags)
 		XLogFileName(lastRestartPointFname, restartTli, restartSegNo,
 					 wal_segment_size);
 
-		shell_archive_cleanup(lastRestartPointFname);
+		RecoveryContext.archive_cleanup_cb(lastRestartPointFname);
 	}
 
 	return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 50b0d1105d..574af1494e 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -22,7 +22,9 @@
 #include "access/xlog.h"
 #include "access/xlog_internal.h"
 #include "access/xlogarchive.h"
+#include "access/xlogrecovery.h"
 #include "common/archive.h"
+#include "fmgr.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/startup.h"
@@ -32,6 +34,15 @@
 #include "storage/ipc.h"
 #include "storage/lwlock.h"
 
+/*
+ * Global context for recovery-related callbacks.
+ */
+RecoveryModuleCallbacks RecoveryContext;
+
+static void LoadRecoveryCallbacks(const char *cmd, const char *cmd_name,
+								  const char *lib, const char *lib_name,
+								  RecoveryModuleCallbacks *cb);
+
 /*
  * Attempt to retrieve the specified file from off-line archival storage.
  * If successful, fill "path" with its complete path (note that this will be
@@ -71,7 +82,7 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 		goto not_available;
 
 	/* In standby mode, restore_command might not be supplied */
-	if (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0)
+	if (recoveryRestoreCommand[0] == '\0' && recoveryRestoreLibrary[0] == '\0')
 		goto not_available;
 
 	/*
@@ -149,14 +160,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 		XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size);
 
 	/*
-	 * Check signals before restore command and reset afterwards.
+	 * Check signals before restore command/library and reset afterwards.
 	 */
 	PreRestoreCommand();
 
 	/*
 	 * Copy xlog from archival storage to XLOGDIR
 	 */
-	ret = shell_restore(xlogfname, xlogpath, lastRestartPointFname);
+	ret = RecoveryContext.restore_cb(xlogfname, xlogpath,
+									 lastRestartPointFname);
 
 	PostRestoreCommand();
 
@@ -603,3 +615,116 @@ XLogArchiveCleanup(const char *xlog)
 	unlink(archiveStatusPath);
 	/* should we complain about failure? */
 }
+
+/*
+ * Functions for loading recovery callbacks into the global RecoveryContext.
+ *
+ * To ensure that we only copy the necessary callbacks into the global context,
+ * we first copy them into a local context before copying the relevant one.
+ * This means that a recovery module's initialization function might be called
+ * multiple times in the same process.  If a module uses this function for
+ * anything beyond returning its callback functions, it must be able to cope
+ * with multiple invocations.
+ */
+
+/*
+ * Loads all the recovery callbacks for the command/library into cb.  This is
+ * intended for use by the functions below to load individual callbacks into
+ * the global RecoveryContext.
+ *
+ * If both the command and library are nonempty, an ERROR will be raised.
+ * cmd_name and lib_name are the GUC names to be used for the corresponding
+ * ERROR message.
+ */
+static void
+LoadRecoveryCallbacks(const char *cmd, const char *cmd_name, const char *lib,
+					  const char *lib_name, RecoveryModuleCallbacks *cb)
+{
+	RecoveryModuleInit init;
+
+	if (cmd[0] != '\0' && lib[0] != '\0')
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("both %s and %s set", cmd_name, lib_name),
+				 errdetail("Only one of %s, %s may be set.",
+						   cmd_name, lib_name)));
+
+	/*
+	 * If the shell command is enabled, use our special initialization
+	 * function.  Otherwise, load the library and call its
+	 * _PG_recovery_module_init().
+	 */
+	if (lib[0] == '\0')
+		init = shell_restore_init;
+	else
+		init = (RecoveryModuleInit)
+			load_external_function(lib, "_PG_recovery_module_init",
+								   false, NULL);
+
+	if (init == NULL)
+		ereport(ERROR,
+				(errmsg("recovery modules have to define the symbol "
+						"_PG_recovery_module_init")));
+
+	(*init) (cb);
+}
+
+/*
+ * Loads only the restore callback into the global RecoveryContext.
+ */
+void
+LoadRestoreLibrary(void)
+{
+	RecoveryModuleCallbacks tmp = {0};
+
+	LoadRecoveryCallbacks(recoveryRestoreCommand, "restore_command",
+						  recoveryRestoreLibrary, "restore_library",
+						  &tmp);
+
+	if (tmp.restore_cb == NULL)
+		ereport(ERROR,
+				(errmsg("recovery modules used for \"restore_library\" must "
+						"register a restore callback")));
+	else
+		RecoveryContext.restore_cb = tmp.restore_cb;
+}
+
+/*
+ * Loads only the archive-cleanup callback into the global RecoveryContext.
+ */
+void
+LoadArchiveCleanupLibrary(void)
+{
+	RecoveryModuleCallbacks tmp = {0};
+
+	LoadRecoveryCallbacks(archiveCleanupCommand, "archive_cleanup_command",
+						  archiveCleanupLibrary, "archive_cleanup_library",
+						  &tmp);
+
+	if (tmp.archive_cleanup_cb == NULL)
+		ereport(ERROR,
+				(errmsg("recovery modules used for \"archive_cleanup_library\" "
+						"must register an archive cleanup callback")));
+	else
+		RecoveryContext.archive_cleanup_cb = tmp.archive_cleanup_cb;
+}
+
+/*
+ * Loads only the recovery-end callback into the global RecoveryContext.
+ */
+void
+LoadRecoveryEndLibrary(void)
+{
+	RecoveryModuleCallbacks tmp = {0};
+
+	LoadRecoveryCallbacks(recoveryEndCommand, "recovery_end_command",
+						  recoveryEndLibrary, "recovery_end_library",
+						  &tmp);
+
+	if (tmp.recovery_end_cb == NULL)
+		ereport(ERROR,
+				(errmsg("recovery modules used for \"recovery_end_library\" "
+						"must register a recovery end callback")));
+	else
+		RecoveryContext.recovery_end_cb = tmp.recovery_end_cb;
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index d5a81f9d83..d140054627 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -80,8 +80,11 @@ const struct config_enum_entry recovery_target_action_options[] = {
 
 /* options formerly taken from recovery.conf for archive recovery */
 char	   *recoveryRestoreCommand = NULL;
+char	   *recoveryRestoreLibrary = NULL;
 char	   *recoveryEndCommand = NULL;
+char	   *recoveryEndLibrary = NULL;
 char	   *archiveCleanupCommand = NULL;
+char	   *archiveCleanupLibrary = NULL;
 RecoveryTargetType recoveryTarget = RECOVERY_TARGET_UNSET;
 bool		recoveryTargetInclusive = true;
 int			recoveryTargetAction = RECOVERY_TARGET_ACTION_PAUSE;
@@ -1059,20 +1062,27 @@ validateRecoveryParameters(void)
 	if (StandbyModeRequested)
 	{
 		if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
-			(recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+			recoveryRestoreCommand[0] == '\0' && recoveryRestoreLibrary[0] == '\0')
 			ereport(WARNING,
-					(errmsg("specified neither primary_conninfo nor restore_command"),
+					(errmsg("specified neither primary_conninfo nor restore_command nor restore_library"),
 					 errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
 	}
 	else
 	{
-		if (recoveryRestoreCommand == NULL ||
-			strcmp(recoveryRestoreCommand, "") == 0)
+		if (recoveryRestoreCommand[0] == '\0' && recoveryRestoreLibrary[0] == '\0')
 			ereport(FATAL,
 					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
-					 errmsg("must specify restore_command when standby mode is not enabled")));
+					 errmsg("must specify restore_command or restore_library when standby mode is not enabled")));
 	}
 
+	/*
+	 * Load the restore and recovery end libraries.  This also checks for
+	 * invalid combinations of the command/library parameters.
+	 */
+	memset(&RecoveryContext, 0, sizeof(RecoveryModuleCallbacks));
+	LoadRestoreLibrary();
+	LoadRecoveryEndLibrary();
+
 	/*
 	 * Override any inconsistent requests. Note that this is a change of
 	 * behaviour in 9.5; prior to this we simply ignored a request to pause if
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 5fc076fc14..9b479b41b4 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -38,6 +38,7 @@
 
 #include "access/xlog.h"
 #include "access/xlog_internal.h"
+#include "access/xlogarchive.h"
 #include "access/xlogrecovery.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -222,6 +223,15 @@ CheckpointerMain(void)
 	 */
 	before_shmem_exit(pgstat_before_server_shutdown, 0);
 
+	/*
+	 * Load the archive cleanup library.  This also checks that at most one of
+	 * archive_cleanup_command, archive_cleanup_library is set.  We do this
+	 * before setting up the exception handler so that any problems result in a
+	 * server crash shortly after startup.
+	 */
+	memset(&RecoveryContext, 0, sizeof(RecoveryModuleCallbacks));
+	LoadArchiveCleanupLibrary();
+
 	/*
 	 * Create a memory context that we will do all our work in.  We do this so
 	 * that we can reset the context during error recovery and thereby avoid
@@ -563,6 +573,19 @@ HandleCheckpointerInterrupts(void)
 		 * because of SIGHUP.
 		 */
 		UpdateSharedMemoryConfig();
+
+		/*
+		 * Since archive_cleanup_command can be changed at runtime, we have to
+		 * validate that only one of the library/command is set after every
+		 * SIGHUP.  Failing recovery seems harsh, so we just warn that the
+		 * shell command will be ignored.
+		 */
+		if (archiveCleanupCommand[0] != '\0' && archiveCleanupLibrary[0] != '\0')
+			ereport(WARNING,
+					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					 errmsg("both archive_cleanup_command and archive_cleanup_library set"),
+					 errdetail("The value of archive_cleanup_command will be ignored."),
+					 errhint("Only one of archive_cleanup_command, archive_cleanup_library may be set.")));
 	}
 	if (ShutdownRequestPending)
 	{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index f99186eab7..5dc830e6c0 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -133,7 +133,8 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
  * Re-read the config file.
  *
  * If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it.  Also, check restore_command/library and
+ * recovery_end_command/library.
  */
 static void
 StartupRereadConfig(void)
@@ -161,6 +162,25 @@ StartupRereadConfig(void)
 
 	if (conninfoChanged || slotnameChanged || tempSlotChanged)
 		StartupRequestWalReceiverRestart();
+
+	/*
+	 * Since restore_command and recovery_end_command can be changed at runtime,
+	 * we have to validate that only one of the library/command is set after
+	 * every SIGHUP.  Failing recovery seems harsh, so we just warn that the
+	 * shell command will be ignored.
+	 */
+	if (recoveryRestoreCommand[0] != '\0' && recoveryRestoreLibrary[0] != '\0')
+		ereport(WARNING,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("both restore_command and restore_library set"),
+				 errdetail("The value of restore_command will be ignored."),
+				 errhint("Only one of restore_command, restore_library may be set.")));
+	if (recoveryEndCommand[0] != '\0' && recoveryEndLibrary[0] != '\0')
+		ereport(WARNING,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("both recovery_end_command and recovery_end_library set"),
+				 errdetail("The value of recovery_end_command will be ignored."),
+				 errhint("Only one of recovery_end_command, recovery_end_library may be set.")));
 }
 
 /* Handle various signals that might be sent to the startup process */
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index a37c9f9844..9aacb584f9 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3764,6 +3764,16 @@ struct config_string ConfigureNamesString[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"restore_library", PGC_POSTMASTER, WAL_ARCHIVE_RECOVERY,
+			gettext_noop("Sets the library that will be called to retrieve an archived WAL file."),
+			gettext_noop("An empty string indicates that \"restore_command\" should be used.")
+		},
+		&recoveryRestoreLibrary,
+		"",
+		NULL, NULL, NULL
+	},
+
 	{
 		{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
 			gettext_noop("Sets the shell command that will be executed at every restart point."),
@@ -3774,6 +3784,16 @@ struct config_string ConfigureNamesString[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"archive_cleanup_library", PGC_POSTMASTER, WAL_ARCHIVE_RECOVERY,
+			gettext_noop("Sets the library that will be executed at every restart point."),
+			gettext_noop("An empty string indicates that \"archive_cleanup_command\" should be used.")
+		},
+		&archiveCleanupLibrary,
+		"",
+		NULL, NULL, NULL
+	},
+
 	{
 		{"recovery_end_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
 			gettext_noop("Sets the shell command that will be executed once at the end of recovery."),
@@ -3784,6 +3804,16 @@ struct config_string ConfigureNamesString[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"recovery_end_library", PGC_POSTMASTER, WAL_ARCHIVE_RECOVERY,
+			gettext_noop("Sets the library that will be executed once at the end of recovery."),
+			gettext_noop("An empty string indicates that \"recovery_end_command\" should be used.")
+		},
+		&recoveryEndLibrary,
+		"",
+		NULL, NULL, NULL
+	},
+
 	{
 		{"recovery_target_timeline", PGC_POSTMASTER, WAL_RECOVERY_TARGET,
 			gettext_noop("Specifies the timeline to recover into."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 5afdeb04de..13aa10b87e 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -269,8 +269,11 @@
 				# placeholders: %p = path of file to restore
 				#               %f = file name only
 				# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = ''		# library to use to restore an archived WAL file
 #archive_cleanup_command = ''	# command to execute at every restartpoint
+#archive_cleanup_library = ''	# library to execute at every restartpoint
 #recovery_end_command = ''	# command to execute at completion of recovery
+#recovery_end_library = ''	# library to execute at completion of recovery
 
 # - Recovery Target -
 
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index e5fc66966b..467f95962b 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -400,5 +400,6 @@ extern PGDLLIMPORT bool ArchiveRecoveryRequested;
 extern PGDLLIMPORT bool InArchiveRecovery;
 extern PGDLLIMPORT bool StandbyMode;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
+extern PGDLLIMPORT char *recoveryRestoreLibrary;
 
 #endif							/* XLOG_INTERNAL_H */
diff --git a/src/include/access/xlogarchive.h b/src/include/access/xlogarchive.h
index 69d002cdeb..c693d200c1 100644
--- a/src/include/access/xlogarchive.h
+++ b/src/include/access/xlogarchive.h
@@ -30,9 +30,44 @@ extern bool XLogArchiveIsReady(const char *xlog);
 extern bool XLogArchiveIsReadyOrDone(const char *xlog);
 extern void XLogArchiveCleanup(const char *xlog);
 
-extern bool shell_restore(const char *file, const char *path,
-						  const char *lastRestartPointFileName);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Recovery module callbacks
+ *
+ * These callback functions should be defined by recovery libraries and
+ * returned via _PG_recovery_module_init().  For more information about the
+ * purpose of each callback, refer to the recovery modules documentation.
+ */
+typedef bool (*RecoveryRestoreCB) (const char *file, const char *path,
+								   const char *lastRestartPointFileName);
+typedef void (*RecoveryArchiveCleanupCB) (const char *lastRestartPointFileName);
+typedef void (*RecoveryEndCB) (const char *lastRestartPointFileName);
+
+typedef struct RecoveryModuleCallbacks
+{
+	RecoveryRestoreCB restore_cb;
+	RecoveryArchiveCleanupCB archive_cleanup_cb;
+	RecoveryEndCB recovery_end_cb;
+} RecoveryModuleCallbacks;
+
+extern RecoveryModuleCallbacks RecoveryContext;
+
+/*
+ * Type of the shared library symbol _PG_recovery_module_init that is looked up
+ * when loading a recovery library.
+ */
+typedef void (*RecoveryModuleInit) (RecoveryModuleCallbacks *cb);
+
+extern PGDLLEXPORT void _PG_recovery_module_init(RecoveryModuleCallbacks *cb);
+
+extern void LoadRestoreLibrary(void);
+extern void LoadArchiveCleanupLibrary(void);
+extern void LoadRecoveryEndLibrary(void);
+
+/*
+ * Since the logic for recovery via a shell command is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern void shell_restore_init(RecoveryModuleCallbacks *cb);
 
 #endif							/* XLOG_ARCHIVE_H */
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index f3398425d8..f73fb12605 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -55,8 +55,11 @@ extern PGDLLIMPORT int recovery_min_apply_delay;
 extern PGDLLIMPORT char *PrimaryConnInfo;
 extern PGDLLIMPORT char *PrimarySlotName;
 extern PGDLLIMPORT char *recoveryRestoreCommand;
+extern PGDLLIMPORT char *recoveryRestoreLibrary;
 extern PGDLLIMPORT char *recoveryEndCommand;
+extern PGDLLIMPORT char *recoveryEndLibrary;
 extern PGDLLIMPORT char *archiveCleanupCommand;
+extern PGDLLIMPORT char *archiveCleanupLibrary;
 
 /* indirectly set via GUC system */
 extern PGDLLIMPORT TransactionId recoveryTargetXid;
-- 
2.25.1

