From 02896517f42d60e8f436ec5d0ab1a55b0ce1a3f9 Mon Sep 17 00:00:00 2001
From: Takashi Menjo <takashi.menjou.vg@hco.ntt.co.jp>
Date: Fri, 24 Jan 2020 13:16:26 +0900
Subject: [PATCH 1/3] Support GUCs for external WAL buffer

To implement non-volatile WAL buffer, we add two new GUCs nvwal_path
and nvwal_size.  Now postgres maps a file at that path onto memory to
use it as WAL buffer.  Note that the buffer is still volatile for now.
---
 configure                                     |  99 +++++++++++
 configure.in                                  |  19 ++
 src/backend/access/transam/Makefile           |   2 +-
 src/backend/access/transam/nv_xlog_buffer.c   |  95 ++++++++++
 src/backend/access/transam/xlog.c             | 164 ++++++++++++++++--
 src/backend/utils/misc/guc.c                  |  23 ++-
 src/backend/utils/misc/postgresql.conf.sample |   2 +
 src/bin/initdb/initdb.c                       |  95 +++++++++-
 src/include/access/nv_xlog_buffer.h           |  71 ++++++++
 src/include/access/xlog.h                     |   2 +
 src/include/pg_config.h.in                    |   6 +
 src/include/utils/guc.h                       |   4 +
 12 files changed, 560 insertions(+), 22 deletions(-)
 create mode 100644 src/backend/access/transam/nv_xlog_buffer.c
 create mode 100644 src/include/access/nv_xlog_buffer.h

diff --git a/configure b/configure
index 54c852aca5..4674419094 100755
--- a/configure
+++ b/configure
@@ -864,6 +864,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_nvwal
 with_gnu_ld
 enable_largefile
 enable_float4_byval
@@ -1570,6 +1571,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-nvwal            use non-volatile WAL buffer (NVWAL)
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
 
 Some influential environment variables:
@@ -8306,6 +8308,40 @@ fi
 
 
 
+#
+# Non-volatile WAL buffer (NVWAL)
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with non-volatile WAL buffer (NVWAL)" >&5
+$as_echo_n "checking whether to build with non-volatile WAL buffer (NVWAL)... " >&6; }
+
+
+
+# Check whether --with-nvwal was given.
+if test "${with_nvwal+set}" = set; then :
+  withval=$with_nvwal;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_NVWAL 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-nvwal option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_nvwal=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_nvwal" >&5
+$as_echo "$with_nvwal" >&6; }
+
 #
 # Elf
 #
@@ -12694,6 +12730,57 @@ fi
 fi
 
 
+# for non-volatile WAL buffer (NVWAL)
+if test "$with_nvwal" = yes; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for pmem_map_file in -lpmem" >&5
+$as_echo_n "checking for pmem_map_file in -lpmem... " >&6; }
+if ${ac_cv_lib_pmem_pmem_map_file+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpmem  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char pmem_map_file ();
+int
+main ()
+{
+return pmem_map_file ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_pmem_pmem_map_file=yes
+else
+  ac_cv_lib_pmem_pmem_map_file=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pmem_pmem_map_file" >&5
+$as_echo "$ac_cv_lib_pmem_pmem_map_file" >&6; }
+if test "x$ac_cv_lib_pmem_pmem_map_file" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBPMEM 1
+_ACEOF
+
+  LIBS="-lpmem $LIBS"
+
+else
+  as_fn_error $? "library 'libpmem' is required for non-volatile WAL buffer (NVWAL)" "$LINENO" 5
+fi
+
+fi
+
 
 ##
 ## Header files
@@ -13373,6 +13460,18 @@ fi
 
 done
 
+fi
+
+# for non-volatile WAL buffer (NVWAL)
+if test "$with_nvwal" = yes ; then
+  ac_fn_c_check_header_mongrel "$LINENO" "libpmem.h" "ac_cv_header_libpmem_h" "$ac_includes_default"
+if test "x$ac_cv_header_libpmem_h" = xyes; then :
+
+else
+  as_fn_error $? "header file <libpmem.h> is required for non-volatile WAL buffer (NVWAL)" "$LINENO" 5
+fi
+
+
 fi
 
 if test "$PORTNAME" = "win32" ; then
diff --git a/configure.in b/configure.in
index 6942f81d1e..d2062d020a 100644
--- a/configure.in
+++ b/configure.in
@@ -964,6 +964,14 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# Non-volatile WAL buffer (NVWAL)
+#
+AC_MSG_CHECKING([whether to build with non-volatile WAL buffer (NVWAL)])
+PGAC_ARG_BOOL(with, nvwal, no, [use non-volatile WAL buffer (NVWAL)],
+              [AC_DEFINE([USE_NVWAL], 1, [Define to 1 to use non-volatile WAL buffer (NVWAL). (--with-nvwal)])])
+AC_MSG_RESULT([$with_nvwal])
+
 #
 # Elf
 #
@@ -1287,6 +1295,12 @@ elif test "$with_uuid" = ossp ; then
 fi
 AC_SUBST(UUID_LIBS)
 
+# for non-volatile WAL buffer (NVWAL)
+if test "$with_nvwal" = yes; then
+  AC_CHECK_LIB(pmem, pmem_map_file, [],
+               [AC_MSG_ERROR([library 'libpmem' is required for non-volatile WAL buffer (NVWAL)])])
+fi
+
 
 ##
 ## Header files
@@ -1467,6 +1481,11 @@ elif test "$with_uuid" = ossp ; then
       [AC_MSG_ERROR([header file <ossp/uuid.h> or <uuid.h> is required for OSSP UUID])])])
 fi
 
+# for non-volatile WAL buffer (NVWAL)
+if test "$with_nvwal" = yes ; then
+  AC_CHECK_HEADER(libpmem.h, [], [AC_MSG_ERROR([header file <libpmem.h> is required for non-volatile WAL buffer (NVWAL)])])
+fi
+
 if test "$PORTNAME" = "win32" ; then
    AC_CHECK_HEADERS(crtdefs.h)
 fi
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47269..addeae9477 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -15,7 +15,7 @@ include $(top_builddir)/src/Makefile.global
 OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
 	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
 	xact.o xlog.o xlogarchive.o xlogfuncs.o \
-	xloginsert.o xlogreader.o xlogutils.o
+	xloginsert.o xlogreader.o xlogutils.o nv_xlog_buffer.o
 
 include $(top_srcdir)/src/backend/common.mk
 
diff --git a/src/backend/access/transam/nv_xlog_buffer.c b/src/backend/access/transam/nv_xlog_buffer.c
new file mode 100644
index 0000000000..cfc6a6376b
--- /dev/null
+++ b/src/backend/access/transam/nv_xlog_buffer.c
@@ -0,0 +1,95 @@
+/*-------------------------------------------------------------------------
+ *
+ * nv_xlog_buffer.c
+ *		PostgreSQL non-volatile WAL buffer
+ *
+ *
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/backend/access/transam/nv_xlog_buffer.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#ifdef USE_NVWAL
+
+#include <libpmem.h>
+#include "access/nv_xlog_buffer.h"
+
+#include "miscadmin.h" /* IsBootstrapProcessingMode */
+#include "common/file_perm.h" /* pg_file_create_mode */
+
+/*
+ * Maps non-volatile WAL buffer on shared memory.
+ *
+ * Returns a mapped address if success; PANICs and never return otherwise.
+ */
+void *
+MapNonVolatileXLogBuffer(const char *fname, Size fsize)
+{
+	void	   *addr;
+	size_t		map_len = 0;
+	int			is_pmem = 0;
+
+	Assert(fname != NULL);
+	Assert(fsize > 0);
+
+	if (IsBootstrapProcessingMode())
+	{
+		/*
+		 * Create and map a new file if we are in bootstrap mode (typically
+		 * executed by initdb).
+		 */
+		addr = pmem_map_file(fname, fsize, PMEM_FILE_CREATE|PMEM_FILE_EXCL,
+							 pg_file_create_mode, &map_len, &is_pmem);
+	}
+	else
+	{
+		/*
+		 * Map an existing file.  The second argument (len) should be zero,
+		 * the third argument (flags) should have neither PMEM_FILE_CREATE nor
+		 * PMEM_FILE_EXCL, and the fourth argument (mode) will be ignored.
+		 */
+		addr = pmem_map_file(fname, 0, 0, 0, &map_len, &is_pmem);
+	}
+
+	if (addr == NULL)
+		elog(PANIC, "could not map non-volatile WAL buffer '%s': %m", fname);
+
+	if (map_len != fsize)
+		elog(PANIC, "size of non-volatile WAL buffer '%s' is invalid; "
+					"expected %zu; actual %zu",
+			 fname, fsize, map_len);
+
+	if (!is_pmem)
+		elog(PANIC, "non-volatile WAL buffer '%s' is not on persistent memory",
+			 fname);
+
+	/*
+	 * Assert page boundary alignment (8KiB as default).  It should pass because
+	 * PMDK considers hugepage boundary alignment (2MiB or 1GiB on x64).
+	 */
+	Assert((uint64) addr % XLOG_BLCKSZ == 0);
+
+	elog(LOG, "non-volatile WAL buffer '%s' is mapped on [%p-%p)",
+		 fname, addr, (char *) addr + map_len);
+	return addr;
+}
+
+void
+UnmapNonVolatileXLogBuffer(void *addr, Size fsize)
+{
+	Assert(addr != NULL);
+
+	if (pmem_unmap(addr, fsize) < 0)
+	{
+		elog(WARNING, "could not unmap non-volatile WAL buffer: %m");
+		return;
+	}
+
+	elog(LOG, "non-volatile WAL buffer unmapped");
+}
+
+#endif
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 77ad765989..eae0c01e3c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -36,6 +36,7 @@
 #include "access/xloginsert.h"
 #include "access/xlogreader.h"
 #include "access/xlogutils.h"
+#include "access/nv_xlog_buffer.h"
 #include "catalog/catversion.h"
 #include "catalog/pg_control.h"
 #include "catalog/pg_database.h"
@@ -848,6 +849,12 @@ static bool InRedo = false;
 /* Have we launched bgwriter during recovery? */
 static bool bgwriterLaunched = false;
 
+/* For non-volatile WAL buffer (NVWAL) */
+char	   *NvwalPath = NULL;	/* a GUC parameter */
+int			NvwalSizeMB = 1024;	/* a direct GUC parameter */
+static Size	NvwalSize = 0;		/* an indirect GUC parameter */
+static bool	NvwalAvail = false;
+
 /* For WALInsertLockAcquire/Release functions */
 static int	MyLockNo = 0;
 static bool holdingAllLocks = false;
@@ -4906,6 +4913,76 @@ check_wal_buffers(int *newval, void **extra, GucSource source)
 	return true;
 }
 
+/*
+ * GUC check_hook for nvwal_path.
+ */
+bool
+check_nvwal_path(char **newval, void **extra, GucSource source)
+{
+#ifndef USE_NVWAL
+	Assert(!NvwalAvail);
+
+	if (**newval != '\0')
+	{
+		GUC_check_errcode(ERRCODE_INVALID_PARAMETER_VALUE);
+		GUC_check_errmsg("nvwal_path is invalid parameter without NVWAL");
+		return false;
+	}
+#endif
+
+	return true;
+}
+
+void
+assign_nvwal_path(const char *newval, void *extra)
+{
+	/* true if not empty; false if empty */
+	NvwalAvail = (bool) (*newval != '\0');
+}
+
+/*
+ * GUC check_hook for nvwal_size.
+ *
+ * It checks the boundary only and DOES NOT check if the size is multiple
+ * of wal_segment_size because the segment size (probably stored in the
+ * control file) have not been set properly here yet.
+ *
+ * See XLOGShmemSize for more validation.
+ */
+bool
+check_nvwal_size(int *newval, void **extra, GucSource source)
+{
+#ifdef USE_NVWAL
+	Size		buf_size;
+	int64		npages;
+
+	Assert(*newval > 0);
+
+	buf_size = (Size) (*newval) * 1024 * 1024;
+	npages = (int64) buf_size / XLOG_BLCKSZ;
+	Assert(npages > 0);
+
+	if (npages > INT_MAX)
+	{
+		/* XLOG_BLCKSZ could be so small that npages exceeds INT_MAX */
+		GUC_check_errcode(ERRCODE_INVALID_PARAMETER_VALUE);
+		GUC_check_errmsg("invalid value for nvwal_size (%dMB): "
+						 "the number of WAL pages too large; "
+						 "buf_size %zu; XLOG_BLCKSZ %d",
+						 *newval, buf_size, (int) XLOG_BLCKSZ);
+		return false;
+	}
+#endif
+
+	return true;
+}
+
+void
+assign_nvwal_size(int newval, void *extra)
+{
+	NvwalSize = (Size) newval * 1024 * 1024;
+}
+
 /*
  * Read the control file, set respective GUCs.
  *
@@ -4934,13 +5011,49 @@ XLOGShmemSize(void)
 {
 	Size		size;
 
+	/*
+	 * If we use non-volatile WAL buffer, we don't use the given wal_buffers.
+	 * Instead, we set it the value based on the size of the file for the
+	 * buffer. This should be done here because of xlblocks array calculation.
+	 */
+	if (NvwalAvail)
+	{
+		char		buf[32];
+		int64		npages;
+
+		Assert(NvwalSizeMB > 0);
+		Assert(NvwalSize > 0);
+		Assert(wal_segment_size > 0);
+		Assert(wal_segment_size % XLOG_BLCKSZ == 0);
+
+		/*
+		 * At last, we can check if the size of non-volatile WAL buffer
+		 * (nvwal_size) is multiple of WAL segment size.
+		 *
+		 * Note that NvwalSize has already been calculated in assign_nvwal_size.
+		 */
+		if (NvwalSize % wal_segment_size != 0)
+		{
+			elog(PANIC,
+				 "invalid value for nvwal_size (%dMB): "
+				 "it should be multiple of WAL segment size; "
+				 "NvwalSize %zu; wal_segment_size %d",
+				 NvwalSizeMB, NvwalSize, wal_segment_size);
+		}
+
+		npages = (int64) NvwalSize / XLOG_BLCKSZ;
+		Assert(npages > 0 && npages <= INT_MAX);
+
+		snprintf(buf, sizeof(buf), "%d", (int) npages);
+		SetConfigOption("wal_buffers", buf, PGC_POSTMASTER, PGC_S_OVERRIDE);
+	}
 	/*
 	 * If the value of wal_buffers is -1, use the preferred auto-tune value.
 	 * This isn't an amazingly clean place to do this, but we must wait till
 	 * NBuffers has received its final value, and must do it before using the
 	 * value of XLOGbuffers to do anything important.
 	 */
-	if (XLOGbuffers == -1)
+	else if (XLOGbuffers == -1)
 	{
 		char		buf[32];
 
@@ -4956,10 +5069,13 @@ XLOGShmemSize(void)
 	size = add_size(size, mul_size(sizeof(WALInsertLockPadded), NUM_XLOGINSERT_LOCKS + 1));
 	/* xlblocks array */
 	size = add_size(size, mul_size(sizeof(XLogRecPtr), XLOGbuffers));
-	/* extra alignment padding for XLOG I/O buffers */
-	size = add_size(size, XLOG_BLCKSZ);
-	/* and the buffers themselves */
-	size = add_size(size, mul_size(XLOG_BLCKSZ, XLOGbuffers));
+	if (!NvwalAvail)
+	{
+		/* extra alignment padding for XLOG I/O buffers */
+		size = add_size(size, XLOG_BLCKSZ);
+		/* and the buffers themselves */
+		size = add_size(size, mul_size(XLOG_BLCKSZ, XLOGbuffers));
+	}
 
 	/*
 	 * Note: we don't count ControlFileData, it comes out of the "slop factor"
@@ -5056,13 +5172,32 @@ XLOGShmemInit(void)
 	}
 
 	/*
-	 * Align the start of the page buffers to a full xlog block size boundary.
-	 * This simplifies some calculations in XLOG insertion. It is also
-	 * required for O_DIRECT.
+	 * Open and memory-map a file for non-volatile XLOG buffer. The PMDK will
+	 * align the start of the buffer to 2-MiB boundary if the size of the
+	 * buffer is larger than or equal to 4 MiB.
 	 */
-	allocptr = (char *) TYPEALIGN(XLOG_BLCKSZ, allocptr);
-	XLogCtl->pages = allocptr;
-	memset(XLogCtl->pages, 0, (Size) XLOG_BLCKSZ * XLOGbuffers);
+	if (NvwalAvail)
+	{
+		/* Logging and error-handling should be done in the function */
+		XLogCtl->pages = MapNonVolatileXLogBuffer(NvwalPath, NvwalSize);
+
+		/*
+		 * Do not memset non-volatile XLOG buffer (XLogCtl->pages) here
+		 * because it would contain records for recovery. We should do so in
+		 * checkpoint after the recovery completes successfully.
+		 */
+	}
+	else
+	{
+		/*
+		 * Align the start of the page buffers to a full xlog block size
+		 * boundary. This simplifies some calculations in XLOG insertion. It
+		 * is also required for O_DIRECT.
+		 */
+		allocptr = (char *) TYPEALIGN(XLOG_BLCKSZ, allocptr);
+		XLogCtl->pages = allocptr;
+		memset(XLogCtl->pages, 0, (Size) XLOG_BLCKSZ * XLOGbuffers);
+	}
 
 	/*
 	 * Do basic initialization of XLogCtl shared data. (StartupXLOG will fill
@@ -8343,6 +8478,13 @@ ShutdownXLOG(int code, Datum arg)
 
 		CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE);
 	}
+
+	/*
+	 * If we use non-volatile XLOG buffer, unmap it.
+	 */
+	if (NvwalAvail)
+		UnmapNonVolatileXLogBuffer(XLogCtl->pages, NvwalSize);
+
 	ShutdownCLOG();
 	ShutdownCommitTs();
 	ShutdownSUBTRANS();
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index f0ed326a1b..39d087d2d1 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2606,7 +2606,7 @@ static struct config_int ConfigureNamesInt[] =
 			GUC_UNIT_XBLOCKS
 		},
 		&XLOGbuffers,
-		-1, -1, (INT_MAX / XLOG_BLCKSZ),
+		-1, -1, INT_MAX,
 		check_wal_buffers, NULL, NULL
 	},
 
@@ -3194,6 +3194,17 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, assign_tcp_user_timeout, show_tcp_user_timeout
 	},
 
+	{
+		{"nvwal_size", PGC_POSTMASTER, WAL_SETTINGS,
+			gettext_noop("Size of non-volatile WAL buffer (NVWAL)."),
+			NULL,
+			GUC_UNIT_MB
+		},
+		&NvwalSizeMB,
+		1024, 1, INT_MAX,
+		check_nvwal_size, assign_nvwal_size, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL
@@ -4199,6 +4210,16 @@ static struct config_string ConfigureNamesString[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"nvwal_path", PGC_POSTMASTER, WAL_SETTINGS,
+			gettext_noop("Path to file for non-volatile WAL buffer (NVWAL)."),
+			NULL
+		},
+		&NvwalPath,
+		"",
+		check_nvwal_path, assign_nvwal_path, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b61e66932c..f77a4a7d0e 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -223,6 +223,8 @@
 #checkpoint_timeout = 5min		# range 30s-1d
 #max_wal_size = 1GB
 #min_wal_size = 80MB
+#nvwal_path = '/path/to/nvwal'
+#nvwal_size = 1GB
 #checkpoint_completion_target = 0.5	# checkpoint target duration, 0.0 - 1.0
 #checkpoint_flush_after = 0		# measured in pages, 0 disables
 #checkpoint_warning = 30s		# 0 disables
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index acf610808e..f08da4da9b 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -144,7 +144,10 @@ static bool show_setting = false;
 static bool data_checksums = false;
 static char *xlog_dir = NULL;
 static char *str_wal_segment_size_mb = NULL;
+static char *nvwal_path = NULL;
+static char *str_nvwal_size_mb = NULL;
 static int	wal_segment_size_mb;
+static int	nvwal_size_mb;
 
 
 /* internal vars */
@@ -1115,14 +1118,78 @@ setup_config(void)
 	conflines = replace_token(conflines, "#port = 5432", repltok);
 #endif
 
-	/* set default max_wal_size and min_wal_size */
-	snprintf(repltok, sizeof(repltok), "min_wal_size = %s",
-			 pretty_wal_size(DEFAULT_MIN_WAL_SEGS));
-	conflines = replace_token(conflines, "#min_wal_size = 80MB", repltok);
-
-	snprintf(repltok, sizeof(repltok), "max_wal_size = %s",
-			 pretty_wal_size(DEFAULT_MAX_WAL_SEGS));
-	conflines = replace_token(conflines, "#max_wal_size = 1GB", repltok);
+	if (nvwal_path != NULL)
+	{
+		int nr_segs;
+
+		if (str_nvwal_size_mb == NULL)
+			nvwal_size_mb = 1024;
+		else
+		{
+			char *endptr;
+
+			/* check that the argument is a number */
+			nvwal_size_mb = strtol(str_nvwal_size_mb, &endptr, 10);
+
+			/* verify that the size of non-volatile WAL buffer is valid */
+			if (endptr == str_nvwal_size_mb || *endptr != '\0')
+			{
+				pg_log_error("argument of --nvwal-size must be a number; "
+							 "str_nvwal_size_mb '%s'",
+							 str_nvwal_size_mb);
+				exit(1);
+			}
+			if (nvwal_size_mb <= 0)
+			{
+				pg_log_error("argument of --nvwal-size must be a positive number; "
+							 "str_nvwal_size_mb '%s'; nvwal_size_mb %d",
+							 str_nvwal_size_mb, nvwal_size_mb);
+				exit(1);
+			}
+			if (nvwal_size_mb % wal_segment_size_mb != 0)
+			{
+				pg_log_error("argument of --nvwal-size must be multiple of WAL segment size; "
+							 "str_nvwal_size_mb '%s'; nvwal_size_mb %d; wal_segment_size_mb %d",
+							 str_nvwal_size_mb, nvwal_size_mb, wal_segment_size_mb);
+				exit(1);
+			}
+		}
+
+		/*
+		 * XXX We set {min_,max_,nv}wal_size to the same value.  Note that
+		 * postgres might bootstrap and run if the three config does not have
+		 * the same value, but have not been tested yet.
+		 */
+		nr_segs = nvwal_size_mb / wal_segment_size_mb;
+
+		snprintf(repltok, sizeof(repltok), "min_wal_size = %s",
+				 pretty_wal_size(nr_segs));
+		conflines = replace_token(conflines, "#min_wal_size = 80MB", repltok);
+
+		snprintf(repltok, sizeof(repltok), "max_wal_size = %s",
+				 pretty_wal_size(nr_segs));
+		conflines = replace_token(conflines, "#max_wal_size = 1GB", repltok);
+
+		snprintf(repltok, sizeof(repltok), "nvwal_path = '%s'",
+				 nvwal_path);
+		conflines = replace_token(conflines,
+								  "#nvwal_path = '/path/to/nvwal'", repltok);
+
+		snprintf(repltok, sizeof(repltok), "nvwal_size = %s",
+				 pretty_wal_size(nr_segs));
+		conflines = replace_token(conflines, "#nvwal_size = 1GB", repltok);
+	}
+	else
+	{
+		/* set default max_wal_size and min_wal_size */
+		snprintf(repltok, sizeof(repltok), "min_wal_size = %s",
+				 pretty_wal_size(DEFAULT_MIN_WAL_SEGS));
+		conflines = replace_token(conflines, "#min_wal_size = 80MB", repltok);
+
+		snprintf(repltok, sizeof(repltok), "max_wal_size = %s",
+				 pretty_wal_size(DEFAULT_MAX_WAL_SEGS));
+		conflines = replace_token(conflines, "#max_wal_size = 1GB", repltok);
+	}
 
 	snprintf(repltok, sizeof(repltok), "lc_messages = '%s'",
 			 escape_quotes(lc_messages));
@@ -2373,6 +2440,8 @@ usage(const char *progname)
 	printf(_("  -W, --pwprompt            prompt for a password for the new superuser\n"));
 	printf(_("  -X, --waldir=WALDIR       location for the write-ahead log directory\n"));
 	printf(_("      --wal-segsize=SIZE    size of WAL segments, in megabytes\n"));
+	printf(_("  -P, --nvwal-path=FILE     path to file for non-volatile WAL buffer (NVWAL)\n"));
+	printf(_("  -Q, --nvwal-size=SIZE     size of NVWAL, in megabytes\n"));
 	printf(_("\nLess commonly used options:\n"));
 	printf(_("  -d, --debug               generate lots of debugging output\n"));
 	printf(_("  -k, --data-checksums      use data page checksums\n"));
@@ -3051,6 +3120,8 @@ main(int argc, char *argv[])
 		{"sync-only", no_argument, NULL, 'S'},
 		{"waldir", required_argument, NULL, 'X'},
 		{"wal-segsize", required_argument, NULL, 12},
+		{"nvwal-path", required_argument, NULL, 'P'},
+		{"nvwal-size", required_argument, NULL, 'Q'},
 		{"data-checksums", no_argument, NULL, 'k'},
 		{"allow-group-access", no_argument, NULL, 'g'},
 		{NULL, 0, NULL, 0}
@@ -3094,7 +3165,7 @@ main(int argc, char *argv[])
 
 	/* process command-line options */
 
-	while ((c = getopt_long(argc, argv, "dD:E:kL:nNU:WA:sST:X:g", long_options, &option_index)) != -1)
+	while ((c = getopt_long(argc, argv, "dD:E:kL:nNU:WA:sST:X:P:Q:g", long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
@@ -3188,6 +3259,12 @@ main(int argc, char *argv[])
 			case 12:
 				str_wal_segment_size_mb = pg_strdup(optarg);
 				break;
+			case 'P':
+				nvwal_path = pg_strdup(optarg);
+				break;
+			case 'Q':
+				str_nvwal_size_mb = pg_strdup(optarg);
+				break;
 			case 'g':
 				SetDataDirectoryCreatePerm(PG_DIR_MODE_GROUP);
 				break;
diff --git a/src/include/access/nv_xlog_buffer.h b/src/include/access/nv_xlog_buffer.h
new file mode 100644
index 0000000000..b58878c92b
--- /dev/null
+++ b/src/include/access/nv_xlog_buffer.h
@@ -0,0 +1,71 @@
+/*
+ * nv_xlog_buffer.h
+ *
+ * PostgreSQL non-volatile WAL buffer
+ *
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/access/nv_xlog_buffer.h
+ */
+#ifndef NV_XLOG_BUFFER_H
+#define NV_XLOG_BUFFER_H
+
+extern void *MapNonVolatileXLogBuffer(const char *fname, Size fsize);
+extern void	UnmapNonVolatileXLogBuffer(void *addr, Size fsize);
+
+#ifdef USE_NVWAL
+#include <libpmem.h>
+
+#define nv_memset_persist	pmem_memset_persist
+#define nv_memcpy_nodrain	pmem_memcpy_nodrain
+#define nv_flush			pmem_flush
+#define nv_drain			pmem_drain
+#define nv_persist			pmem_persist
+
+#else
+void *
+MapNonVolatileXLogBuffer(const char *fname, Size fsize)
+{
+	return NULL;
+}
+
+void
+UnmapNonVolatileXLogBuffer(void *addr, Size fsize)
+{
+	return;
+}
+
+static inline void *
+nv_memset_persist(void *pmemdest, int c, size_t len)
+{
+	return NULL;
+}
+
+static inline void *
+nv_memcpy_nodrain(void *pmemdest, const void *src,
+				  size_t len)
+{
+	return NULL;
+}
+
+static inline void
+nv_flush(void *pmemdest, size_t len)
+{
+	return;
+}
+
+static inline void
+nv_drain(void)
+{
+	return;
+}
+
+static inline void
+nv_persist(const void *addr, size_t len)
+{
+	return;
+}
+
+#endif							/* USE_NVWAL */
+#endif							/* NV_XLOG_BUFFER_H */
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index d519252aad..bc09fa104c 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -129,6 +129,8 @@ extern int	recoveryTargetAction;
 extern int	recovery_min_apply_delay;
 extern char *PrimaryConnInfo;
 extern char *PrimarySlotName;
+extern char *NvwalPath;
+extern int  NvwalSizeMB;
 
 /* indirectly set via GUC system */
 extern TransactionId recoveryTargetXid;
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 512213aa32..bd2b434d93 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -356,6 +356,9 @@
 /* Define to 1 if you have the `pam' library (-lpam). */
 #undef HAVE_LIBPAM
 
+/* Define to 1 if you have the `pmem' library (-lpmem). */
+#undef HAVE_LIBPMEM
+
 /* Define if you have a function readline library */
 #undef HAVE_LIBREADLINE
 
@@ -932,6 +935,9 @@
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
+/* Define to 1 to use non-volatile WAL buffer (NVWAL). (--with-nvwal) */
+#undef USE_NVWAL
+
 /* Define to build with OpenSSL support. (--with-openssl) */
 #undef USE_OPENSSL
 
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index a93ed77c9c..3bd4bbb872 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -432,6 +432,10 @@ extern void assign_search_path(const char *newval, void *extra);
 
 /* in access/transam/xlog.c */
 extern bool check_wal_buffers(int *newval, void **extra, GucSource source);
+extern bool check_nvwal_path(char **newval, void **extra, GucSource source);
+extern void assign_nvwal_path(const char *newval, void *extra);
+extern bool check_nvwal_size(int *newval, void **extra, GucSource source);
+extern void assign_nvwal_size(int newval, void *extra);
 extern void assign_xlog_sync_method(int new_sync_method, void *extra);
 
 #endif							/* GUC_H */
-- 
2.20.1

