From 6fc7ba367e58382675a9ba8ba0e5d18fdd86ccfd Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathandbossart@gmail.com>
Date: Wed, 15 Feb 2023 10:36:00 -0800
Subject: [PATCH v13 4/7] refactor code for restoring via shell

---
 src/backend/Makefile                      |   2 +-
 src/backend/access/transam/timeline.c     |  12 +-
 src/backend/access/transam/xlog.c         |  50 ++++-
 src/backend/access/transam/xlogarchive.c  | 167 ++++-----------
 src/backend/access/transam/xlogrecovery.c |   3 +-
 src/backend/meson.build                   |   1 +
 src/backend/postmaster/startup.c          |  16 +-
 src/backend/restore/Makefile              |  18 ++
 src/backend/restore/meson.build           |   5 +
 src/backend/restore/shell_restore.c       | 246 ++++++++++++++++++++++
 src/include/Makefile                      |   2 +-
 src/include/access/xlogarchive.h          |   9 +-
 src/include/meson.build                   |   1 +
 src/include/postmaster/startup.h          |   1 +
 src/include/restore/shell_restore.h       |  26 +++
 src/tools/pgindent/typedefs.list          |   1 +
 16 files changed, 408 insertions(+), 152 deletions(-)
 create mode 100644 src/backend/restore/Makefile
 create mode 100644 src/backend/restore/meson.build
 create mode 100644 src/backend/restore/shell_restore.c
 create mode 100644 src/include/restore/shell_restore.h

diff --git a/src/backend/Makefile b/src/backend/Makefile
index e4bf0fe9c0..23812e9a6a 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -20,7 +20,7 @@ include $(top_builddir)/src/Makefile.global
 SUBDIRS = access archive backup bootstrap catalog parser commands executor \
 	foreign lib libpq \
 	main nodes optimizer partitioning port postmaster \
-	regex replication rewrite \
+	regex replication restore rewrite \
 	statistics storage tcop tsearch utils $(top_builddir)/src/timezone \
 	jit
 
diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c
index 94e152694e..b3e1c18c65 100644
--- a/src/backend/access/transam/timeline.c
+++ b/src/backend/access/transam/timeline.c
@@ -59,7 +59,8 @@ restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end)
 			continue;
 
 		TLHistoryFileName(histfname, tli);
-		if (RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false))
+		if (RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false,
+								ARCHIVE_TYPE_TIMELINE_HISTORY))
 			KeepFileRestoredFromArchive(path, histfname);
 	}
 }
@@ -97,7 +98,8 @@ readTimeLineHistory(TimeLineID targetTLI)
 	{
 		TLHistoryFileName(histfname, targetTLI);
 		fromArchive =
-			RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false);
+			RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false,
+								ARCHIVE_TYPE_TIMELINE_HISTORY);
 	}
 	else
 		TLHistoryFilePath(path, targetTLI);
@@ -232,7 +234,8 @@ existsTimeLineHistory(TimeLineID probeTLI)
 	if (ArchiveRecoveryRequested)
 	{
 		TLHistoryFileName(histfname, probeTLI);
-		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false);
+		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false,
+							ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS);
 	}
 	else
 		TLHistoryFilePath(path, probeTLI);
@@ -334,7 +337,8 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI,
 	if (ArchiveRecoveryRequested)
 	{
 		TLHistoryFileName(histfname, parentTLI);
-		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false);
+		RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false,
+							ARCHIVE_TYPE_TIMELINE_HISTORY);
 	}
 	else
 		TLHistoryFilePath(path, parentTLI);
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 543d4d897a..8e1e71c256 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -84,6 +84,7 @@
 #include "replication/snapbuild.h"
 #include "replication/walreceiver.h"
 #include "replication/walsender.h"
+#include "restore/shell_restore.h"
 #include "storage/bufmgr.h"
 #include "storage/fd.h"
 #include "storage/ipc.h"
@@ -692,6 +693,7 @@ static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli);
 static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos);
 static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos);
 static uint64 XLogRecPtrToBytePos(XLogRecPtr ptr);
+static void GetOldestRestartPointFileName(char *fname);
 
 static void WALInsertLockAcquire(void);
 static void WALInsertLockAcquireExclusive(void);
@@ -4890,12 +4892,18 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
 {
 	/*
 	 * Execute the recovery_end_command, if any.
+	 *
+	 * The command is provided the archive file cutoff point for use during log
+	 * shipping replication.  All files earlier than this point can be deleted
+	 * from the archive, though there is no requirement to do so.
 	 */
-	if (recoveryEndCommand && strcmp(recoveryEndCommand, "") != 0)
-		ExecuteRecoveryCommand(recoveryEndCommand,
-							   "recovery_end_command",
-							   true,
-							   WAIT_EVENT_RECOVERY_END_COMMAND);
+	if (shell_recovery_end_configured())
+	{
+		char		lastRestartPointFname[MAXFNAMELEN];
+
+		GetOldestRestartPointFileName(lastRestartPointFname);
+		shell_recovery_end(lastRestartPointFname);
+	}
 
 	/*
 	 * We switched to a new timeline. Clean up segments on the old timeline.
@@ -7310,12 +7318,18 @@ CreateRestartPoint(int flags)
 
 	/*
 	 * Finally, execute archive_cleanup_command, if any.
+	 *
+	 * The command is provided the archive file cutoff point for use during log
+	 * shipping replication.  All files earlier than this point can be deleted
+	 * from the archive, though there is no requirement to do so.
 	 */
-	if (archiveCleanupCommand && strcmp(archiveCleanupCommand, "") != 0)
-		ExecuteRecoveryCommand(archiveCleanupCommand,
-							   "archive_cleanup_command",
-							   false,
-							   WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND);
+	if (shell_archive_cleanup_configured())
+	{
+		char		lastRestartPointFname[MAXFNAMELEN];
+
+		GetOldestRestartPointFileName(lastRestartPointFname);
+		shell_archive_cleanup(lastRestartPointFname);
+	}
 
 	return true;
 }
@@ -8892,6 +8906,22 @@ GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli)
 	LWLockRelease(ControlFileLock);
 }
 
+/*
+ * Returns the WAL file name for the last checkpoint or restartpoint.  This is
+ * the oldest WAL file that we still need if we have to restart recovery.
+ */
+static void
+GetOldestRestartPointFileName(char *fname)
+{
+	XLogRecPtr	restartRedoPtr;
+	TimeLineID	restartTli;
+	XLogSegNo	restartSegNo;
+
+	GetOldestRestartPoint(&restartRedoPtr, &restartTli);
+	XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size);
+	XLogFileName(fname, restartTli, restartSegNo, wal_segment_size);
+}
+
 /* Thin wrapper around ShutdownWalRcv(). */
 void
 XLogShutdownWalRcv(void)
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 41684418b6..4b45ea8753 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,12 +23,12 @@
 #include "access/xlog_internal.h"
 #include "access/xlogarchive.h"
 #include "common/archive.h"
-#include "common/percentrepl.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/startup.h"
 #include "postmaster/pgarch.h"
 #include "replication/walsender.h"
+#include "restore/shell_restore.h"
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/lwlock.h"
@@ -54,12 +54,11 @@
 bool
 RestoreArchivedFile(char *path, const char *xlogfname,
 					const char *recovername, off_t expectedSize,
-					bool cleanupEnabled)
+					bool cleanupEnabled, ArchiveType archive_type)
 {
 	char		xlogpath[MAXPGPATH];
-	char	   *xlogRestoreCmd;
 	char		lastRestartPointFname[MAXPGPATH];
-	int			rc;
+	bool		ret = false;
 	struct stat stat_buf;
 	XLogSegNo	restartSegNo;
 	XLogRecPtr	restartRedoPtr;
@@ -73,8 +72,21 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 		goto not_available;
 
 	/* In standby mode, restore_command might not be supplied */
-	if (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0)
-		goto not_available;
+	switch (archive_type)
+	{
+		case ARCHIVE_TYPE_WAL_SEGMENT:
+			if (!shell_restore_file_configured())
+				goto not_available;
+			break;
+		case ARCHIVE_TYPE_TIMELINE_HISTORY:
+			if (!shell_restore_file_configured())
+				goto not_available;
+			break;
+		case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
+			if (!shell_restore_file_configured())
+				goto not_available;
+			break;
+	}
 
 	/*
 	 * When doing archive recovery, we always prefer an archived log file even
@@ -150,39 +162,33 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 	else
 		XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size);
 
-	/* Build the restore command to execute */
-	xlogRestoreCmd = BuildRestoreCommand(recoveryRestoreCommand,
-										 xlogpath, xlogfname,
-										 lastRestartPointFname);
-
-	ereport(DEBUG3,
-			(errmsg_internal("executing restore command \"%s\"",
-							 xlogRestoreCmd)));
-
-	fflush(NULL);
-	pgstat_report_wait_start(WAIT_EVENT_RESTORE_COMMAND);
-
 	/*
-	 * PreRestoreCommand() informs the SIGTERM handler for the startup process
-	 * that it should proc_exit() right away.  This is done for the duration of
-	 * the system() call because there isn't a good way to break out while it
-	 * is executing.  Since we might call proc_exit() in a signal handler, it
-	 * is best to put any additional logic before or after the
-	 * PreRestoreCommand()/PostRestoreCommand() section.
+	 * To ensure we are responsive to server shutdown, check for shutdown
+	 * requests before and after restoring a file.  If there is one, we exit
+	 * right away.
 	 */
-	PreRestoreCommand();
+	HandleStartupProcShutdownRequests();
 
 	/*
 	 * Copy xlog from archival storage to XLOGDIR
 	 */
-	rc = system(xlogRestoreCmd);
-
-	PostRestoreCommand();
+	switch (archive_type)
+	{
+		case ARCHIVE_TYPE_WAL_SEGMENT:
+			ret = shell_restore_wal_segment(xlogfname, xlogpath,
+											lastRestartPointFname);
+			break;
+		case ARCHIVE_TYPE_TIMELINE_HISTORY:
+			ret = shell_restore_timeline_history(xlogfname, xlogpath);
+			break;
+		case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
+			ret = shell_restore_timeline_history(xlogfname, xlogpath);
+			break;
+	}
 
-	pgstat_report_wait_end();
-	pfree(xlogRestoreCmd);
+	HandleStartupProcShutdownRequests();
 
-	if (rc == 0)
+	if (ret)
 	{
 		/*
 		 * command apparently succeeded, but let's make sure the file is
@@ -238,37 +244,6 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 		}
 	}
 
-	/*
-	 * Remember, we rollforward UNTIL the restore fails so failure here is
-	 * just part of the process... that makes it difficult to determine
-	 * whether the restore failed because there isn't an archive to restore,
-	 * or because the administrator has specified the restore program
-	 * incorrectly.  We have to assume the former.
-	 *
-	 * However, if the failure was due to any sort of signal, it's best to
-	 * punt and abort recovery.  (If we "return false" here, upper levels will
-	 * assume that recovery is complete and start up the database!) It's
-	 * essential to abort on child SIGINT and SIGQUIT, because per spec
-	 * system() ignores SIGINT and SIGQUIT while waiting; if we see one of
-	 * those it's a good bet we should have gotten it too.
-	 *
-	 * On SIGTERM, assume we have received a fast shutdown request, and exit
-	 * cleanly. It's pure chance whether we receive the SIGTERM first, or the
-	 * child process. If we receive it first, the signal handler will call
-	 * proc_exit, otherwise we do it here. If we or the child process received
-	 * SIGTERM for any other reason than a fast shutdown request, postmaster
-	 * will perform an immediate shutdown when it sees us exiting
-	 * unexpectedly.
-	 *
-	 * We treat hard shell errors such as "command not found" as fatal, too.
-	 */
-	if (wait_result_is_signal(rc, SIGTERM))
-		proc_exit(1);
-
-	ereport(wait_result_is_any_signal(rc, true) ? FATAL : DEBUG2,
-			(errmsg("could not restore file \"%s\" from archive: %s",
-					xlogfname, wait_result_to_str(rc))));
-
 not_available:
 
 	/*
@@ -282,74 +257,6 @@ not_available:
 	return false;
 }
 
-/*
- * Attempt to execute an external shell command during recovery.
- *
- * 'command' is the shell command to be executed, 'commandName' is a
- * human-readable name describing the command emitted in the logs. If
- * 'failOnSignal' is true and the command is killed by a signal, a FATAL
- * error is thrown. Otherwise a WARNING is emitted.
- *
- * This is currently used for recovery_end_command and archive_cleanup_command.
- */
-void
-ExecuteRecoveryCommand(const char *command, const char *commandName,
-					   bool failOnSignal, uint32 wait_event_info)
-{
-	char	   *xlogRecoveryCmd;
-	char		lastRestartPointFname[MAXPGPATH];
-	int			rc;
-	XLogSegNo	restartSegNo;
-	XLogRecPtr	restartRedoPtr;
-	TimeLineID	restartTli;
-
-	Assert(command && commandName);
-
-	/*
-	 * Calculate the archive file cutoff point for use during log shipping
-	 * replication. All files earlier than this point can be deleted from the
-	 * archive, though there is no requirement to do so.
-	 */
-	GetOldestRestartPoint(&restartRedoPtr, &restartTli);
-	XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size);
-	XLogFileName(lastRestartPointFname, restartTli, restartSegNo,
-				 wal_segment_size);
-
-	/*
-	 * construct the command to be executed
-	 */
-	xlogRecoveryCmd = replace_percent_placeholders(command, commandName, "r", lastRestartPointFname);
-
-	ereport(DEBUG3,
-			(errmsg_internal("executing %s \"%s\"", commandName, command)));
-
-	/*
-	 * execute the constructed command
-	 */
-	fflush(NULL);
-	pgstat_report_wait_start(wait_event_info);
-	rc = system(xlogRecoveryCmd);
-	pgstat_report_wait_end();
-
-	pfree(xlogRecoveryCmd);
-
-	if (rc != 0)
-	{
-		/*
-		 * If the failure was due to any sort of signal, it's best to punt and
-		 * abort recovery.  See comments in RestoreArchivedFile().
-		 */
-		ereport((failOnSignal && wait_result_is_any_signal(rc, true)) ? FATAL : WARNING,
-		/*------
-		   translator: First %s represents a postgresql.conf parameter name like
-		  "recovery_end_command", the 2nd is the value of that parameter, the
-		  third an already translated error message. */
-				(errmsg("%s \"%s\": %s", commandName,
-						command, wait_result_to_str(rc))));
-	}
-}
-
-
 /*
  * A file was restored from the archive under a temporary filename (path),
  * and now we want to keep it. Rename it under the permanent filename in
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index dbe9394762..f0e1007f92 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -4106,7 +4106,8 @@ XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
 			if (!RestoreArchivedFile(path, xlogfname,
 									 "RECOVERYXLOG",
 									 wal_segment_size,
-									 InRedo))
+									 InRedo,
+									 ARCHIVE_TYPE_WAL_SEGMENT))
 				return -1;
 			break;
 
diff --git a/src/backend/meson.build b/src/backend/meson.build
index ccfc382fcf..bb3ddfd38b 100644
--- a/src/backend/meson.build
+++ b/src/backend/meson.build
@@ -26,6 +26,7 @@ subdir('port')
 subdir('postmaster')
 subdir('regex')
 subdir('replication')
+subdir('restore')
 subdir('rewrite')
 subdir('statistics')
 subdir('storage')
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index 0e7de26bc2..adce0ffcef 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -198,8 +198,7 @@ HandleStartupProcInterrupts(void)
 	/*
 	 * Check if we were requested to exit without finishing recovery.
 	 */
-	if (shutdown_requested)
-		proc_exit(1);
+	HandleStartupProcShutdownRequests();
 
 	/*
 	 * Emergency bailout if postmaster has died.  This is to avoid the
@@ -223,6 +222,16 @@ HandleStartupProcInterrupts(void)
 		ProcessLogMemoryContextInterrupt();
 }
 
+/*
+ * If there is a pending shutdown request, exit.
+ */
+void
+HandleStartupProcShutdownRequests(void)
+{
+	if (shutdown_requested)
+		proc_exit(1);
+}
+
 
 /* --------------------------------
  *		signal handler routines
@@ -298,8 +307,7 @@ PreRestoreCommand(void)
 	 * shutdown request received just before this.
 	 */
 	in_restore_command = true;
-	if (shutdown_requested)
-		proc_exit(1);
+	HandleStartupProcShutdownRequests();
 }
 
 void
diff --git a/src/backend/restore/Makefile b/src/backend/restore/Makefile
new file mode 100644
index 0000000000..983d8e980f
--- /dev/null
+++ b/src/backend/restore/Makefile
@@ -0,0 +1,18 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for src/backend/restore
+#
+# IDENTIFICATION
+#    src/backend/restore/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/restore
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = \
+	shell_restore.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/restore/meson.build b/src/backend/restore/meson.build
new file mode 100644
index 0000000000..2b27cfced0
--- /dev/null
+++ b/src/backend/restore/meson.build
@@ -0,0 +1,5 @@
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+backend_sources += files(
+  'shell_restore.c'
+)
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
new file mode 100644
index 0000000000..fcd5475c88
--- /dev/null
+++ b/src/backend/restore/shell_restore.c
@@ -0,0 +1,246 @@
+/*-------------------------------------------------------------------------
+ *
+ * shell_restore.c
+ *
+ * These restore functions use a user-specified shell command (e.g., the
+ * restore_command GUC).
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/restore/shell_restore.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <signal.h>
+
+#include "access/xlog.h"
+#include "access/xlogrecovery.h"
+#include "access/xlog_internal.h"
+#include "common/archive.h"
+#include "common/percentrepl.h"
+#include "postmaster/startup.h"
+#include "restore/shell_restore.h"
+#include "storage/ipc.h"
+#include "utils/wait_event.h"
+
+static bool shell_restore_file(const char *file, const char *path,
+							   const char *lastRestartPointFileName);
+static void ExecuteRecoveryCommand(const char *command,
+								   const char *commandName,
+								   bool failOnSignal,
+								   uint32 wait_event_info,
+								   const char *lastRestartPointFileName);
+
+/*
+ * Attempt to execute a shell-based restore command to retrieve a WAL segment.
+ *
+ * Returns true if the command has succeeded, false otherwise.
+ */
+bool
+shell_restore_wal_segment(const char *file, const char *path,
+						  const char *lastRestartPointFileName)
+{
+	return shell_restore_file(file, path, lastRestartPointFileName);
+}
+
+/*
+ * Attempt to execute a shell-based restore command to retrieve a timeline
+ * history file.
+ *
+ * Returns true if the command has succeeded, false otherwise.
+ */
+bool
+shell_restore_timeline_history(const char *file, const char *path)
+{
+	char		lastRestartPointFname[MAXPGPATH];
+
+	XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size);
+	return shell_restore_file(file, path, lastRestartPointFname);
+}
+
+/*
+ * Check whether restore_command is supplied.
+ */
+bool
+shell_restore_file_configured(void)
+{
+	return recoveryRestoreCommand[0] != '\0';
+}
+
+/*
+ * Attempt to execute a shell-based restore command to retrieve a file.
+ *
+ * Returns true if the command has succeeded, false otherwise.
+ */
+static bool
+shell_restore_file(const char *file, const char *path,
+				   const char *lastRestartPointFileName)
+{
+	char	   *cmd;
+	int			rc;
+
+	/* Build the restore command to execute */
+	cmd = BuildRestoreCommand(recoveryRestoreCommand, path, file,
+							  lastRestartPointFileName);
+
+	ereport(DEBUG3,
+			(errmsg_internal("executing restore command \"%s\"", cmd)));
+
+	fflush(NULL);
+	pgstat_report_wait_start(WAIT_EVENT_RESTORE_COMMAND);
+
+	/*
+	 * Check signals before restore command and reset afterwards.
+	 * PreRestoreCommand() informs the SIGTERM handler for the startup process
+	 * that it should proc_exit() right away.  This is done for the duration of
+	 * the system() call because there isn't a good way to break out while it
+	 * is executing.  Since we might call proc_exit() in a signal handler, it
+	 * is best to put any additional logic before or after the
+	 * PreRestoreCommand()/PostRestoreCommand() section.
+	 */
+	PreRestoreCommand();
+
+	/*
+	 * Copy xlog from archival storage to XLOGDIR
+	 */
+	rc = system(cmd);
+
+	PostRestoreCommand();
+
+	pgstat_report_wait_end();
+	pfree(cmd);
+
+	/*
+	 * Remember, we rollforward UNTIL the restore fails so failure here is
+	 * just part of the process... that makes it difficult to determine
+	 * whether the restore failed because there isn't an archive to restore,
+	 * or because the administrator has specified the restore program
+	 * incorrectly.  We have to assume the former.
+	 *
+	 * However, if the failure was due to any sort of signal, it's best to
+	 * punt and abort recovery.  (If we "return false" here, upper levels will
+	 * assume that recovery is complete and start up the database!) It's
+	 * essential to abort on child SIGINT and SIGQUIT, because per spec
+	 * system() ignores SIGINT and SIGQUIT while waiting; if we see one of
+	 * those it's a good bet we should have gotten it too.
+	 *
+	 * On SIGTERM, assume we have received a fast shutdown request, and exit
+	 * cleanly. It's pure chance whether we receive the SIGTERM first, or the
+	 * child process. If we receive it first, the signal handler will call
+	 * proc_exit, otherwise we do it here. If we or the child process received
+	 * SIGTERM for any other reason than a fast shutdown request, postmaster
+	 * will perform an immediate shutdown when it sees us exiting
+	 * unexpectedly.
+	 *
+	 * We treat hard shell errors such as "command not found" as fatal, too.
+	 */
+	if (rc != 0)
+	{
+		if (wait_result_is_signal(rc, SIGTERM))
+			proc_exit(1);
+
+		ereport(wait_result_is_any_signal(rc, true) ? FATAL : DEBUG2,
+				(errmsg("could not restore file \"%s\" from archive: %s",
+						file, wait_result_to_str(rc))));
+	}
+
+	return (rc == 0);
+}
+
+/*
+ * Check whether archive_cleanup_command is supplied.
+ */
+bool
+shell_archive_cleanup_configured(void)
+{
+	return archiveCleanupCommand[0] != '\0';
+}
+
+/*
+ * Attempt to execute a shell-based archive cleanup command.
+ */
+void
+shell_archive_cleanup(const char *lastRestartPointFileName)
+{
+	ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
+						   false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
+						   lastRestartPointFileName);
+}
+
+/*
+ * Check whether recovery_end_command is supplied.
+ */
+bool
+shell_recovery_end_configured(void)
+{
+	return recoveryEndCommand[0] != '\0';
+}
+
+/*
+ * Attempt to execute a shell-based end-of-recovery command.
+ */
+void
+shell_recovery_end(const char *lastRestartPointFileName)
+{
+	ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
+						   WAIT_EVENT_RECOVERY_END_COMMAND,
+						   lastRestartPointFileName);
+}
+
+/*
+ * Attempt to execute an external shell command during recovery.
+ *
+ * 'command' is the shell command to be executed, 'commandName' is a
+ * human-readable name describing the command emitted in the logs. If
+ * 'failOnSignal' is true and the command is killed by a signal, a FATAL
+ * error is thrown. Otherwise a WARNING is emitted.
+ *
+ * This is currently used for recovery_end_command and archive_cleanup_command.
+ */
+static void
+ExecuteRecoveryCommand(const char *command, const char *commandName,
+					   bool failOnSignal, uint32 wait_event_info,
+					   const char *lastRestartPointFileName)
+{
+	char	   *xlogRecoveryCmd;
+	int			rc;
+
+	Assert(command && commandName);
+
+	/*
+	 * construct the command to be executed
+	 */
+	xlogRecoveryCmd = replace_percent_placeholders(command, commandName, "r",
+												   lastRestartPointFileName);
+
+	ereport(DEBUG3,
+			(errmsg_internal("executing %s \"%s\"", commandName, command)));
+
+	/*
+	 * execute the constructed command
+	 */
+	fflush(NULL);
+	pgstat_report_wait_start(wait_event_info);
+	rc = system(xlogRecoveryCmd);
+	pgstat_report_wait_end();
+
+	pfree(xlogRecoveryCmd);
+
+	if (rc != 0)
+	{
+		/*
+		 * If the failure was due to any sort of signal, it's best to punt and
+		 * abort recovery.  See comments in shell_restore().
+		 */
+		ereport((failOnSignal && wait_result_is_any_signal(rc, true)) ? FATAL : WARNING,
+		/*------
+		   translator: First %s represents a postgresql.conf parameter name like
+		  "recovery_end_command", the 2nd is the value of that parameter, the
+		  third an already translated error message. */
+				(errmsg("%s \"%s\": %s", commandName,
+						command, wait_result_to_str(rc))));
+	}
+}
diff --git a/src/include/Makefile b/src/include/Makefile
index 56576dcf5c..bbc8080845 100644
--- a/src/include/Makefile
+++ b/src/include/Makefile
@@ -20,7 +20,7 @@ all: pg_config.h pg_config_ext.h pg_config_os.h
 SUBDIRS = access archive bootstrap catalog commands common datatype \
 	executor fe_utils foreign jit \
 	lib libpq mb nodes optimizer parser partitioning postmaster \
-	regex replication rewrite \
+	regex replication restore rewrite \
 	statistics storage tcop snowball snowball/libstemmer tsearch \
 	tsearch/dicts utils port port/atomics port/win32 port/win32_msvc \
 	port/win32_msvc/sys port/win32/arpa port/win32/netinet \
diff --git a/src/include/access/xlogarchive.h b/src/include/access/xlogarchive.h
index 31ff206034..b9bd0a93cd 100644
--- a/src/include/access/xlogarchive.h
+++ b/src/include/access/xlogarchive.h
@@ -17,9 +17,16 @@
 
 #include "access/xlogdefs.h"
 
+typedef enum ArchiveType
+{
+	ARCHIVE_TYPE_WAL_SEGMENT,
+	ARCHIVE_TYPE_TIMELINE_HISTORY,
+	ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS
+} ArchiveType;
+
 extern bool RestoreArchivedFile(char *path, const char *xlogfname,
 								const char *recovername, off_t expectedSize,
-								bool cleanupEnabled);
+								bool cleanupEnabled, ArchiveType archive_type);
 extern void ExecuteRecoveryCommand(const char *command, const char *commandName,
 								   bool failOnSignal, uint32 wait_event_info);
 extern void KeepFileRestoredFromArchive(const char *path, const char *xlogfname);
diff --git a/src/include/meson.build b/src/include/meson.build
index 33c0a5562c..abaa529462 100644
--- a/src/include/meson.build
+++ b/src/include/meson.build
@@ -149,6 +149,7 @@ header_subdirs = [
   'postmaster',
   'regex',
   'replication',
+  'restore',
   'rewrite',
   'statistics',
   'storage',
diff --git a/src/include/postmaster/startup.h b/src/include/postmaster/startup.h
index 6a2e4c4526..09d3f89c69 100644
--- a/src/include/postmaster/startup.h
+++ b/src/include/postmaster/startup.h
@@ -26,6 +26,7 @@
 extern PGDLLIMPORT int log_startup_progress_interval;
 
 extern void HandleStartupProcInterrupts(void);
+extern void HandleStartupProcShutdownRequests(void);
 extern void StartupProcessMain(void) pg_attribute_noreturn();
 extern void PreRestoreCommand(void);
 extern void PostRestoreCommand(void);
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
new file mode 100644
index 0000000000..570588e493
--- /dev/null
+++ b/src/include/restore/shell_restore.h
@@ -0,0 +1,26 @@
+/*-------------------------------------------------------------------------
+ *
+ * shell_restore.h
+ *		Exports for restoring via shell.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/restore/shell_restore.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _SHELL_RESTORE_H
+#define _SHELL_RESTORE_H
+
+extern bool shell_restore_file_configured(void);
+extern bool shell_restore_wal_segment(const char *file, const char *path,
+									  const char *lastRestartPointFileName);
+extern bool shell_restore_timeline_history(const char *file, const char *path);
+
+extern bool shell_archive_cleanup_configured(void);
+extern void shell_archive_cleanup(const char *lastRestartPointFileName);
+
+extern bool shell_recovery_end_configured(void);
+extern void shell_recovery_end(const char *lastRestartPointFileName);
+
+#endif							/* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 86a9303bf5..acdc29c0ef 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -132,6 +132,7 @@ ArchiveModuleState
 ArchiveOpts
 ArchiveShutdownCB
 ArchiveStreamState
+ArchiveType
 ArchiverOutput
 ArchiverStage
 ArrayAnalyzeExtraData
-- 
2.25.1

