From 1a69fe5bd62a573427426e2305679562d72090a9 Mon Sep 17 00:00:00 2001
From: Bruce Momjian <bruce@momjian.us>
Date: Fri, 25 Jun 2021 16:56:51 -0400
Subject: [PATCH v2 05/12] cfe-05-crypto_over_cfe-04-common squash commit

---
 doc/src/sgml/config.sgml    | 120 ++++++++--
 src/backend/crypto/Makefile |  18 ++
 src/backend/crypto/kmgr.c   | 438 ++++++++++++++++++++++++++++++++++++
 src/common/kmgr_utils.c     |   3 +-
 src/include/crypto/kmgr.h   |  25 ++
 5 files changed, 588 insertions(+), 16 deletions(-)
 create mode 100644 src/backend/crypto/Makefile
 create mode 100644 src/backend/crypto/kmgr.c
 create mode 100644 src/include/crypto/kmgr.h

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 559eb898a9..4e95f7e22a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1558,19 +1558,31 @@ include_dir 'conf.d'
         mechanism is used.
        </para>
        <para>
-        The command must print the passphrase to the standard output and exit
-        with code 0.  In the parameter value, <literal>%p</literal> is
-        replaced by a prompt string.  (Write <literal>%%</literal> for a
-        literal <literal>%</literal>.)  Note that the prompt string will
-        probably contain whitespace, so be sure to quote adequately.  A single
-        newline is stripped from the end of the output if present.
+        The command must print the passphrase to the standard output
+        and exit with code 0.  It can prompt from the terminal if
+        <option>--authprompt</option> is used.  In the parameter
+        value, <literal>%R</literal> is replaced by a file descriptor
+        number opened to the terminal that started the server.  A file
+        descriptor is only available if enabled at server start via
+        <option>-R</option>.  If <literal>%R</literal> is specified and
+        no file descriptor is available, the server will not start.  Value
+        <literal>%p</literal> is replaced by a pre-defined prompt string.
+        (Write <literal>%%</literal> for a literal <literal>%</literal>.)
+        Note that the prompt string will probably contain whitespace,
+        so be sure to quote its use adequately.  Newlines are stripped
+        from the end of the output if present.
        </para>
+
        <para>
-        The command does not actually have to prompt the user for a
-        passphrase.  It can read it from a file, obtain it from a keychain
-        facility, or similar.  It is up to the user to make sure the chosen
-        mechanism is adequately secure.
+        Sample scripts can be found in
+        <filename>$SHAREDIR/auth_commands</filename>,
+        where <literal>$SHAREDIR</literal> means the
+        <productname>PostgreSQL</productname> installation's shared-data
+        directory, often <filename>/usr/local/share/postgresql</filename>
+        (use <command>pg_config --sharedir</command> to determine it if
+        you're not sure).
        </para>
+
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
         file or on the server command line.
@@ -1592,10 +1604,12 @@ include_dir 'conf.d'
         parameter is off (the default), then
         <varname>ssl_passphrase_command</varname> will be ignored during a
         reload and the SSL configuration will not be reloaded if a passphrase
-        is needed.  That setting is appropriate for a command that requires a
-        TTY for prompting, which might not be available when the server is
-        running.  Setting this parameter to on might be appropriate if the
-        passphrase is obtained from a file, for example.
+        is needed.  This setting is appropriate for a command that requires a
+        terminal for prompting, which will likely not be available when the server is
+        running.  (<option>--authprompt</option> closes the terminal file
+        descriptor soon after server start.)   Setting this parameter on
+        might be appropriate, for example, if the passphrase is obtained
+        from a file.
        </para>
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
@@ -2794,7 +2808,9 @@ include_dir 'conf.d'
         <varname>max_wal_senders</varname> is non-zero.
         Note that changing <varname>wal_level</varname> to
         <literal>minimal</literal> makes previous base backups unusable
-        for point-in-time recovery and standby servers.
+        for point-in-time recovery and standby servers, which may
+        lead to data loss.  Cluster file encryption also does not support
+        <varname>wal_level</varname> <literal>minimal</literal>.
        </para>
        <para>
         In <literal>logical</literal> level, the same information is logged as
@@ -8483,6 +8499,64 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv;
     </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-encryption">
+    <title>Cluster File Encryption</title>
+
+    <variablelist>
+     <varlistentry id="guc-cluster-key-command" xreflabel="cluster_key_command">
+      <term><varname>cluster_key_command</varname> (<type>string</type>)
+      <indexterm>
+       <primary><varname>cluster_key_command</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        This option specifies an external command to obtain the cluster-level
+        key for cluster file encryption during server initialization and
+        server start.
+       </para>
+       <para>
+        The command must print the cluster key to the standard
+        output as 64 hexadecimal characters, and exit with code 0.
+        The command can prompt for the passphrase or PIN from the
+        terminal if <option>--authprompt</option> is used.  In the
+        parameter value, <literal>%R</literal> is replaced by a file
+        descriptor number opened to the terminal that started the server.
+        A file descriptor is only available if enabled at server start
+        via <option>-R</option>.  If <literal>%R</literal> is specified
+        and no file descriptor is available, the server will not start.
+        Value <literal>%p</literal> is replaced by a pre-defined
+        prompt string.  Value <literal>%d</literal> is replaced by the
+        directory containing the keys;  this is useful if the command
+        must create files with the keys, e.g., to store a cluster-level
+        key encrypted by a key stored in a hardware security module.
+        (Write <literal>%%</literal> for a literal <literal>%</literal>.)
+        Note that the prompt string will probably contain whitespace,
+        so be sure to quote its use adequately.  Newlines are stripped
+        from the end of the output if present.
+       </para>
+
+       <para>
+        Sample script can be found in
+        <filename>$SHAREDIR/auth_commands</filename>,
+        where <literal>$SHAREDIR</literal> means the
+        <productname>PostgreSQL</productname> installation's shared-data
+        directory, often <filename>/usr/local/share/postgresql</filename>
+        (use <command>pg_config --sharedir</command> to determine it if
+        you're not sure).
+       </para>
+
+       <para>
+        This parameter can only be set by
+        <application>initdb</application>, in the
+        <filename>postgresql.conf</filename> file, or on the server
+        command line.
+       </para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+   </sect1>
+
    <sect1 id="runtime-config-client">
     <title>Client Connection Defaults</title>
 
@@ -10508,6 +10582,22 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-file-encryption-method" xreflabel="file_encryption_method">
+      <term><varname>file_encryption_method</varname> (<type>boolean</type>)
+      <indexterm>
+       <primary>Cluster file encryption method</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Reports the cluster file
+        encryption method.  See <xref
+        linkend="app-initdb-cluster-key-command"/> for more
+        information.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-data-directory-mode" xreflabel="data_directory_mode">
       <term><varname>data_directory_mode</varname> (<type>integer</type>)
       <indexterm>
diff --git a/src/backend/crypto/Makefile b/src/backend/crypto/Makefile
new file mode 100644
index 0000000000..c27362029d
--- /dev/null
+++ b/src/backend/crypto/Makefile
@@ -0,0 +1,18 @@
+#-------------------------------------------------------------------------
+#
+# Makefile
+#    Makefile for src/backend/crypto
+#
+# IDENTIFICATION
+#    src/backend/crypto/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/crypto
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = \
+	kmgr.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/crypto/kmgr.c b/src/backend/crypto/kmgr.c
new file mode 100644
index 0000000000..4b74961323
--- /dev/null
+++ b/src/backend/crypto/kmgr.c
@@ -0,0 +1,438 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr.c
+ *	 Cluster file encryption routines
+ *
+ * Cluster file encryption is enabled if user requests it during initdb.
+ * During bootstrap, we generate data encryption keys, wrap them with the
+ * cluster-level key, and store them into each file located at KMGR_DIR.
+ * During startup, we decrypt all internal keys and load them to the shared
+ * memory.  Internal keys in the shared memory are read-only.  The wrapping
+ * and unwrapping key routines require the OpenSSL library.
+ *
+ * Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/crypto/kmgr.c
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+
+#include "access/xlog.h"
+#include "common/file_perm.h"
+#include "common/kmgr_utils.h"
+#include "common/sha2.h"
+#include "access/xlog.h"
+#include "common/controldata_utils.h"
+#include "crypto/kmgr.h"
+#include "postmaster/postmaster.h"
+#include "storage/copydir.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/shmem.h"
+#include "utils/builtins.h"
+#include "utils/memutils.h"
+/* Struct stores file encryption keys in plaintext format */
+typedef struct KmgrShmemData
+{
+	CryptoKey	intlKeys[KMGR_NUM_DATA_KEYS];
+} KmgrShmemData;
+static KmgrShmemData *KmgrShmem;
+
+/* GUC variables */
+char	   *cluster_key_command = NULL;
+
+CryptoKey	bootstrap_keys[KMGR_NUM_DATA_KEYS];
+
+extern char *bootstrap_old_key_datadir;
+extern int	bootstrap_file_encryption_method;
+
+static void bzeroKmgrKeys(int status, Datum arg);
+static void KmgrWriteCryptoKeys(const char *dir, unsigned char **keys, int *key_lens);
+static CryptoKey *generate_crypto_key(int len);
+
+/*
+ * This function must be called ONCE during initdb.  It creates the DEK
+ * files wrapped with the KEK supplied by kmgr_run_cluster_key_command().
+ * There is also an option for the keys to be copied from another cluster.
+ */
+void
+BootStrapKmgr(void)
+{
+	char		live_path[MAXPGPATH];
+	unsigned char *keys_wrap[KMGR_NUM_DATA_KEYS];
+	int			key_lens[KMGR_NUM_DATA_KEYS];
+	char		cluster_key_hex[ALLOC_KMGR_CLUSTER_KEY_LEN];
+	int			cluster_key_hex_len;
+	unsigned char cluster_key[KMGR_CLUSTER_KEY_LEN];
+
+	if (!FileEncryptionEnabled)
+		return;
+
+#ifndef USE_OPENSSL
+	ereport(ERROR,
+			(errcode(ERRCODE_CONFIG_FILE_ERROR),
+			 (errmsg("cluster file encryption is not supported because OpenSSL is not supported by this build"),
+			  errhint("Compile with --with-openssl to use this feature."))));
+#endif
+
+	/*
+	 * There are too many optimizations for wal_level=minimal that don't set
+	 * LSNs for permanent tables, so just disallow it.
+	 */
+	if (!XLogIsNeeded())
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIG_FILE_ERROR),
+				 (errmsg("cluster file encryption is not supported with a wal_level of \"minimal\""))));
+
+	snprintf(live_path, sizeof(live_path), "%s/%s", DataDir, LIVE_KMGR_DIR);
+
+	/*
+	 * Copy cluster file encryption keys from an old cluster? This is useful
+	 * for pg_upgrade upgrades where the copied database files are already
+	 * encrypted using the old cluster's DEK keys.
+	 */
+	if (bootstrap_old_key_datadir != NULL)
+	{
+		char		old_key_dir[MAXPGPATH];
+
+		snprintf(old_key_dir, sizeof(old_key_dir), "%s/%s",
+				 bootstrap_old_key_datadir, LIVE_KMGR_DIR);
+		copydir(old_key_dir, LIVE_KMGR_DIR, true);
+	}
+	/* create an empty directory */
+	else
+	{
+		if (mkdir(LIVE_KMGR_DIR, pg_dir_create_mode) < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not create cluster file encryption directory \"%s\": %m",
+							LIVE_KMGR_DIR)));
+	}
+
+	/*
+	 * Get key encryption key (KEK) from the cluster_key command.  The cluster
+	 * key command might need to check for the existence of files in the live
+	 * directory, e.g., PIV, so run this _after_ copying the directory in
+	 * place.
+	 */
+	cluster_key_hex_len = kmgr_run_cluster_key_command(cluster_key_command,
+													   cluster_key_hex,
+													   ALLOC_KMGR_CLUSTER_KEY_LEN,
+													   live_path, terminal_fd);
+
+	/* decode supplied hex */
+	if (hex_decode(cluster_key_hex, cluster_key_hex_len,
+					  (char *) cluster_key) !=
+		KMGR_CLUSTER_KEY_LEN)
+		ereport(ERROR,
+				(errmsg("cluster key must be %d hexadecimal characters",
+						KMGR_CLUSTER_KEY_LEN * 2)));
+
+	/* We are not in copy mode?  Generate new cluster file encryption keys. */
+	if (bootstrap_old_key_datadir == NULL)
+	{
+		unsigned char *bootstrap_keys_wrap[KMGR_NUM_DATA_KEYS];
+		int			key_lens[KMGR_NUM_DATA_KEYS];
+		PgCipherCtx *cluster_key_ctx;
+
+		/* Create KEK encryption context */
+		cluster_key_ctx = pg_cipher_ctx_create(PG_CIPHER_AES_KWP, cluster_key,
+											   KMGR_CLUSTER_KEY_LEN, true);
+		if (!cluster_key_ctx)
+			elog(ERROR, "could not initialize encryption context");
+
+		/* Wrap data encryption keys (DEK) using the key encryption key (KEK) */
+		for (int id = 0; id < KMGR_NUM_DATA_KEYS; id++)
+		{
+			CryptoKey  *key;
+
+			/* generate a DEK */
+			key = generate_crypto_key(
+									  encryption_methods[bootstrap_file_encryption_method].bit_length / 8);
+
+			/* output generated random string as hex, for testing */
+			{
+				char		str[MAXPGPATH];
+				int			out_len;
+
+				out_len = hex_encode((char *) (key->key), key->klen,
+										str);
+				str[out_len] = '\0';
+			}
+
+			bootstrap_keys_wrap[id] = palloc0(KMGR_MAX_KEY_LEN_BYTES +
+											  pg_cipher_blocksize(cluster_key_ctx));
+
+			/* wrap DEK with KEK */
+			if (!kmgr_wrap_data_key(cluster_key_ctx, key, bootstrap_keys_wrap[id], &(key_lens[id])))
+			{
+				pg_cipher_ctx_free(cluster_key_ctx);
+				elog(ERROR, "failed to wrap data encryption key");
+			}
+
+			/* remove DEK from memory */
+			explicit_bzero(key, sizeof(CryptoKey));
+		}
+
+		/* Write data encryption keys to the disk */
+		KmgrWriteCryptoKeys(LIVE_KMGR_DIR, bootstrap_keys_wrap, key_lens);
+
+		pg_cipher_ctx_free(cluster_key_ctx);
+	}
+
+	/*
+	 * We are either decrypting keys we copied from an old cluster, or
+	 * decrypting keys we just wrote above --- either way, we decrypt them
+	 * here and store them in a file-scoped variable for use in later
+	 * encrypting during bootstrap mode.
+	 */
+
+	/* Get the crypto keys from the live directory */
+	kmgr_read_wrapped_data_keys(LIVE_KMGR_DIR, keys_wrap, key_lens);
+
+	if (!kmgr_verify_cluster_key(cluster_key, keys_wrap, key_lens, bootstrap_keys))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("supplied cluster key does not match expected cluster_key")));
+
+	/* bzero DEK on exit */
+	on_proc_exit(bzeroKmgrKeys, 0);
+
+	/* bzero KEK */
+	explicit_bzero(cluster_key_hex, cluster_key_hex_len);
+	explicit_bzero(cluster_key, KMGR_CLUSTER_KEY_LEN);
+}
+
+/* Report shared-memory space needed by KmgrShmem */
+Size
+KmgrShmemSize(void)
+{
+	if (!FileEncryptionEnabled)
+		return 0;
+
+	return MAXALIGN(sizeof(KmgrShmemData));
+}
+
+/* Allocate and initialize key manager memory */
+void
+KmgrShmemInit(void)
+{
+	bool		found;
+
+	if (!FileEncryptionEnabled)
+		return;
+
+	KmgrShmem = (KmgrShmemData *) ShmemInitStruct("File encryption key manager",
+												  KmgrShmemSize(), &found);
+
+	/* bzero DEK on exit */
+	on_shmem_exit(bzeroKmgrKeys, 0);
+}
+
+/*
+ * Get cluster key and verify it, then get the data encryption keys.
+ * This function is called by postmaster at startup time.
+ */
+void
+InitializeKmgr(void)
+{
+	unsigned char *keys_wrap[KMGR_NUM_DATA_KEYS];
+	int			key_lens[KMGR_NUM_DATA_KEYS];
+	char		cluster_key_hex[ALLOC_KMGR_CLUSTER_KEY_LEN];
+	int			cluster_key_hex_len;
+	struct stat buffer;
+	char		live_path[MAXPGPATH];
+	unsigned char cluster_key[KMGR_CLUSTER_KEY_LEN];
+
+#ifndef USE_OPENSSL
+	ereport(ERROR,
+			(errcode(ERRCODE_CONFIG_FILE_ERROR),
+			 (errmsg("cluster file encryption is not supported because OpenSSL is not supported by this build"),
+			  errhint("Compile with --with-openssl to use this feature."))));
+#endif
+
+	/*
+	 * There are too many optimizations for wal_level=minimal that don't set
+	 * LSNs for permanent tables, so just disallow it.
+	 */
+	if (!XLogIsNeeded())
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIG_FILE_ERROR),
+				 (errmsg("cluster file encryption is not supported with a wal_level of \"minimal\""))));
+
+	elog(DEBUG1, "starting up cluster file encryption manager");
+
+	if (stat(KMGR_DIR, &buffer) != 0 || !S_ISDIR(buffer.st_mode))
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 (errmsg("cluster file encryption directory %s is missing", KMGR_DIR))));
+
+	if (stat(KMGR_DIR_PID, &buffer) == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 (errmsg("cluster had a pg_alterckey failure that needs repair or pg_alterckey is running"),
+				  errhint("Run pg_alterckey --repair or wait for it to complete."))));
+
+	/*
+	 * We want OLD deleted since it allows access to the data encryption keys
+	 * using the old cluster key.  If NEW exists, it means either the new
+	 * directory is partly written, or NEW wasn't renamed to LIVE --- in
+	 * either case, it needs to be repaired.  See src/bin/pg_alterckey/README
+	 * for more details.
+	 */
+	if (stat(OLD_KMGR_DIR, &buffer) == 0 || stat(NEW_KMGR_DIR, &buffer) == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 (errmsg("cluster had a pg_alterckey failure that needs repair"),
+				  errhint("Run pg_alterckey --repair."))));
+
+	/* If OLD, NEW, and LIVE do not exist, there is a serious problem. */
+	if (stat(LIVE_KMGR_DIR, &buffer) != 0 || !S_ISDIR(buffer.st_mode))
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 (errmsg("cluster has no data encryption keys"))));
+
+	/* Get the cluster key (KEK) */
+	snprintf(live_path, sizeof(live_path), "%s/%s", DataDir, LIVE_KMGR_DIR);
+	cluster_key_hex_len = kmgr_run_cluster_key_command(cluster_key_command,
+													   cluster_key_hex,
+													   ALLOC_KMGR_CLUSTER_KEY_LEN,
+													   live_path, terminal_fd);
+
+	/* decode supplied hex */
+	if (hex_decode(cluster_key_hex, cluster_key_hex_len,
+					  (char *) cluster_key) !=
+		KMGR_CLUSTER_KEY_LEN)
+		ereport(ERROR,
+				(errmsg("cluster key must be %d hexadecimal characters",
+						KMGR_CLUSTER_KEY_LEN * 2)));
+
+	/* Load wrapped DEKs from their files into an array */
+	kmgr_read_wrapped_data_keys(LIVE_KMGR_DIR, keys_wrap, key_lens);
+
+	/*
+	 * Verify cluster key and store the unwrapped data encryption keys in
+	 * shared memory.
+	 */
+	if (!kmgr_verify_cluster_key(cluster_key, keys_wrap, key_lens, KmgrShmem->intlKeys))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("supplied cluster key does not match expected cluster key")));
+
+	/* Check that retrieved key lengths match controldata length. */
+	for (int id = 0; id < KMGR_NUM_DATA_KEYS; id++)
+		if (KmgrShmem->intlKeys[id].klen * 8 !=
+			encryption_methods[GetFileEncryptionMethod()].bit_length)
+		{
+			char		path[MAXPGPATH];
+
+			CryptoKeyFilePath(path, DataDir, id);
+
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					 errmsg("data encryption key %s of length %d does not match controldata key length %d",
+							path, KmgrShmem->intlKeys[id].klen * 8,
+							encryption_methods[GetFileEncryptionMethod()].bit_length)));
+		}
+
+	/* bzero KEK */
+	explicit_bzero(cluster_key_hex, cluster_key_hex_len);
+	explicit_bzero(cluster_key, KMGR_CLUSTER_KEY_LEN);
+}
+
+static void
+bzeroKmgrKeys(int status, Datum arg)
+{
+	if (IsBootstrapProcessingMode())
+		explicit_bzero(bootstrap_keys, sizeof(bootstrap_keys));
+	else
+		explicit_bzero(KmgrShmem->intlKeys, sizeof(KmgrShmem->intlKeys));
+}
+
+/* return requested DEK */
+const CryptoKey *
+KmgrGetKey(int id)
+{
+	Assert(id < KMGR_NUM_DATA_KEYS);
+
+	return (const CryptoKey *) (IsBootstrapProcessingMode() ?
+								&(bootstrap_keys[id]) : &(KmgrShmem->intlKeys[id]));
+}
+
+/* Generate a DEK inside a CryptoKey */
+static CryptoKey *
+generate_crypto_key(int len)
+{
+	CryptoKey  *newkey;
+
+	Assert(len <= KMGR_MAX_KEY_LEN);
+	newkey = (CryptoKey *) palloc0(sizeof(CryptoKey));
+
+	newkey->klen = len;
+
+	if (!pg_strong_random(newkey->key, len))
+		elog(ERROR, "failed to generate new file encryption key");
+
+	return newkey;
+}
+
+/*
+ * Write the DEKs to the disk.
+ */
+static void
+KmgrWriteCryptoKeys(const char *dir, unsigned char **keys, int *key_lens)
+{
+	elog(DEBUG2, "writing data encryption keys wrapped using the cluster key");
+
+	for (int i = 0; i < KMGR_NUM_DATA_KEYS; i++)
+	{
+		int			fd;
+		char		path[MAXPGPATH];
+
+		CryptoKeyFilePath(path, dir, i);
+
+		if ((fd = BasicOpenFile(path, O_RDWR | O_CREAT | O_EXCL | PG_BINARY)) < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not open file \"%s\": %m",
+							path)));
+
+		errno = 0;
+		pgstat_report_wait_start(WAIT_EVENT_KEY_FILE_WRITE);
+		if (write(fd, keys[i], key_lens[i]) != key_lens[i])
+		{
+			/* if write didn't set errno, assume problem is no disk space */
+			if (errno == 0)
+				errno = ENOSPC;
+
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not write file \"%s\": %m",
+							path)));
+		}
+		pgstat_report_wait_end();
+
+		pgstat_report_wait_start(WAIT_EVENT_KEY_FILE_SYNC);
+		if (pg_fsync(fd) != 0)
+			ereport(PANIC,
+					(errcode_for_file_access(),
+					 errmsg("could not fsync file \"%s\": %m",
+							path)));
+		pgstat_report_wait_end();
+
+		if (close(fd) != 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not close file \"%s\": %m",
+							path)));
+	}
+}
diff --git a/src/common/kmgr_utils.c b/src/common/kmgr_utils.c
index 6c50cb093b..92b6441186 100644
--- a/src/common/kmgr_utils.c
+++ b/src/common/kmgr_utils.c
@@ -30,11 +30,12 @@
 #endif
 #include "common/cryptohash.h"
 #include "common/file_perm.h"
-#include "common/hex.h"
 #include "common/string.h"
 #include "crypto/kmgr.h"
 #include "lib/stringinfo.h"
 #include "storage/fd.h"
+#include "postgres.h"
+#include "utils/builtins.h"
 
 #ifndef FRONTEND
 #include "pgstat.h"
diff --git a/src/include/crypto/kmgr.h b/src/include/crypto/kmgr.h
new file mode 100644
index 0000000000..ec341c70b9
--- /dev/null
+++ b/src/include/crypto/kmgr.h
@@ -0,0 +1,25 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr.h
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/crypto/kmgr.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef KMGR_H
+#define KMGR_H
+
+#include "common/kmgr_utils.h"
+
+/* GUC parameters */
+extern char *cluster_key_command;
+
+extern Size KmgrShmemSize(void);
+extern void KmgrShmemInit(void);
+extern void BootStrapKmgr(void);
+extern void InitializeKmgr(void);
+extern const CryptoKey *KmgrGetKey(int id);
+
+#endif							/* KMGR_H */
-- 
2.31.1

