From 70183f1ba1eb33e5c279a6d22a56fcaebdbfbb97 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sat, 14 Dec 2019 16:22:15 -0600
Subject: [PATCH v8 4/6] pg_ls_tmpdir to show directories recursively

See also 9cd92d1a33699f86aa53d44ab04cc3eb50c18d11

Discussion
https://www.postgresql.org/message-id/flat/20191213053931.GV2082@telsasoft.com
https://www.postgresql.org/message-id/flat/20191227170220.GE12890@telsasoft.com

Need catversion bump
---
 doc/src/sgml/func.sgml                    |  14 +-
 src/backend/utils/adt/genfile.c           | 167 ++++++++++++++++------
 src/include/catalog/pg_proc.dat           |   8 +-
 src/test/regress/input/tablespace.source  |   5 +
 src/test/regress/output/tablespace.source |   8 ++
 5 files changed, 147 insertions(+), 55 deletions(-)

diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index fc4d7f0f78..234c9c1699 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -21382,8 +21382,9 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup());
        </entry>
        <entry><type>setof record</type></entry>
        <entry>
-        List the name, size, and last modification time of files in the
-        temporary directory for <parameter>tablespace</parameter>.  If
+        For the temporary directory within <parameter>tablespace</parameter>,
+        list each file's name, size, last modification time, and boolean
+        indicating if it is a directory.  Directories are shown recursively.  If
         <parameter>tablespace</parameter> is not provided, the
         <literal>pg_default</literal> tablespace is used.  Access is granted
         to members of the <literal>pg_monitor</literal> role and may be
@@ -21482,14 +21483,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup());
     <primary>pg_ls_tmpdir</primary>
    </indexterm>
    <para>
-    <function>pg_ls_tmpdir</function> returns the name, size, and last modified
-    time (mtime) of each file in the temporary file directory for the specified
-    <parameter>tablespace</parameter>.  If <parameter>tablespace</parameter> is
+    <function>pg_ls_tmpdir</function> lists each file in the temporary file
+    directory for the specified <parameter>tablespace</parameter>, along with
+    its size, last modified time (mtime) and boolean indicating if the file is a
+    directory.  Directories are used for temporary files shared by parallel
+    processes, and are shown recursively.  If <parameter>tablespace</parameter> is
     not provided, the <literal>pg_default</literal> tablespace is used.  By
     default only superusers and members of the <literal>pg_monitor</literal>
     role can use this function.  Access may be granted to others using
     <command>GRANT</command>.
-    Filenames beginning with a dot, directories, and other special files are not shown.
    </para>
 
    <indexterm>
diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c
index 897b11a77d..a534d75dd8 100644
--- a/src/backend/utils/adt/genfile.c
+++ b/src/backend/utils/adt/genfile.c
@@ -35,11 +35,23 @@
 #include "utils/syscache.h"
 #include "utils/timestamp.h"
 
+enum dir_action {
+	DIR_HIDE,
+	DIR_ONLY,
+	DIR_DESCEND,
+};
+
 typedef struct
 {
-	char	   *location;
-	DIR		   *dirdesc;
+	/* Stack of opened dirs */
+	List		*location;
+	List		*dirdesc;
 	bool		include_dot_dirs;
+
+	/* Used in ls_dir_files: */
+	enum dir_action dir_action;
+	char		*path;
+	struct stat	stat;
 } directory_fctx;
 
 
@@ -469,10 +481,9 @@ pg_ls_dir(PG_FUNCTION_ARGS)
 		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
 
 		fctx = palloc(sizeof(directory_fctx));
-		fctx->location = convert_and_check_filename(PG_GETARG_TEXT_PP(0));
-
+		fctx->location = lappend(NIL, convert_and_check_filename(PG_GETARG_TEXT_PP(0)));
+		fctx->dirdesc = lappend(NIL, AllocateDir(linitial(fctx->location)));
 		fctx->include_dot_dirs = include_dot_dirs;
-		fctx->dirdesc = AllocateDir(fctx->location);
 
 		if (!fctx->dirdesc)
 		{
@@ -485,7 +496,7 @@ pg_ls_dir(PG_FUNCTION_ARGS)
 				ereport(ERROR,
 						(errcode_for_file_access(),
 						 errmsg("could not open directory \"%s\": %m",
-								fctx->location)));
+								(char*)linitial(fctx->location))));
 		}
 		funcctx->user_fctx = fctx;
 		MemoryContextSwitchTo(oldcontext);
@@ -494,7 +505,7 @@ pg_ls_dir(PG_FUNCTION_ARGS)
 	funcctx = SRF_PERCALL_SETUP();
 	fctx = (directory_fctx *) funcctx->user_fctx;
 
-	while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL)
+	while ((de = ReadDir(linitial(fctx->dirdesc), linitial(fctx->location))) != NULL)
 	{
 		if (!fctx->include_dot_dirs &&
 			(strcmp(de->d_name, ".") == 0 ||
@@ -504,7 +515,9 @@ pg_ls_dir(PG_FUNCTION_ARGS)
 		SRF_RETURN_NEXT(funcctx, CStringGetTextDatum(de->d_name));
 	}
 
-	FreeDir(fctx->dirdesc);
+	FreeDir(linitial(fctx->dirdesc));
+	list_free(fctx->dirdesc);
+	list_free_deep(fctx->location);
 
 	SRF_RETURN_DONE(funcctx);
 }
@@ -522,12 +535,84 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS)
 	return pg_ls_dir(fcinfo);
 }
 
-/* Generic function to return a directory listing of files */
+/*
+ * Update fctx->path and stat with next filename.
+ * Directories are descended into.
+ * The current dir to read from is at fctx[fctx->depth].
+ */
+static int
+populate_paths(directory_fctx *fctx, FuncCallContext *funcctx)
+{
+	struct dirent *de;
+
+	for (;;)
+	{
+		char		path[MAXPGPATH];
+		DIR			*dirdesc = llast(fctx->dirdesc);
+		char		*location = llast(fctx->location);
+
+		Assert(list_length(fctx->dirdesc) == list_length(fctx->location));
+
+		if ((de = ReadDir(dirdesc, location)) == NULL)
+		{
+			/*
+			 * Read to the end of the dir on the top of the stack, now move to
+			 * the next dir.
+			 */
+			if (list_length(fctx->dirdesc) == 1)
+				return 0;
+			FreeDir(llast(fctx->dirdesc));
+			fctx->dirdesc = list_delete_last(fctx->dirdesc);
+			pfree(location);
+			fctx->location = list_delete_last(fctx->location);
+			continue;
+		}
+
+		/* Skip hidden files */
+		if (de->d_name[0] == '.')
+			continue;
+
+		/* Get the file info */
+		snprintf(path, sizeof(path), "%s/%s", location, de->d_name);
+		if (stat(path, &fctx->stat) < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not stat file \"%s\": %m", path)));
+
+		/* Ignore anything but regular files, or dirs, if requested */
+		if (S_ISDIR(fctx->stat.st_mode))
+		{
+			if (fctx->dir_action == DIR_HIDE)
+				continue;
+			else if (fctx->dir_action == DIR_DESCEND)
+			{
+				MemoryContext oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+				fctx->location = lappend(fctx->location, pstrdup(path));
+				fctx->dirdesc = lappend(fctx->dirdesc, AllocateDir(path));
+				MemoryContextSwitchTo(oldcontext);
+			}
+		} else if (!S_ISREG(fctx->stat.st_mode))
+			continue;
+
+		/* Do not show the initial dir or slash */
+		fctx->path = pstrdup(path + 1 + strlen(linitial(fctx->location)));
+		return 1;
+	}
+}
+
+/*
+ * Generic function to return a directory listing of files
+ *
+ * if missing_ok, then fail silently if the dir doesn't exist, else error.
+ *
+ * if dir_action!=DIR_HIDE, then include a 4th column indicating whether path
+ * is a directory, and recurse if DIR_DESCEND.
+ */
 static Datum
-pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
+pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok,
+		enum dir_action dir_action)
 {
 	FuncCallContext *funcctx;
-	struct dirent *de;
 	directory_fctx *fctx;
 
 	if (SRF_IS_FIRSTCALL())
@@ -540,19 +625,24 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 
 		fctx = palloc(sizeof(directory_fctx));
 
-		tupdesc = CreateTemplateTupleDesc(3);
+		tupdesc = CreateTemplateTupleDesc(dir_action == DIR_HIDE ? 3 : 4);
 		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "name",
 						   TEXTOID, -1, 0);
 		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "size",
 						   INT8OID, -1, 0);
 		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "modification",
 						   TIMESTAMPTZOID, -1, 0);
+		if (dir_action != DIR_HIDE)
+			TupleDescInitEntry(tupdesc, (AttrNumber) 4, "isdir",
+						   BOOLOID, -1, 0);
+
 		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
 
-		fctx->location = pstrdup(dir);
-		fctx->dirdesc = AllocateDir(fctx->location);
+		fctx->location = lappend(NIL, pstrdup(dir));
+		fctx->dirdesc = lappend(NIL, AllocateDir(dir));
+		fctx->dir_action = dir_action;
 
-		if (!fctx->dirdesc)
+		if (!linitial(fctx->dirdesc))
 		{
 			if (missing_ok && errno == ENOENT)
 			{
@@ -563,7 +653,7 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 				ereport(ERROR,
 						(errcode_for_file_access(),
 						 errmsg("could not open directory \"%s\": %m",
-								fctx->location)));
+								dir)));
 		}
 
 		funcctx->user_fctx = fctx;
@@ -573,39 +663,26 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 	funcctx = SRF_PERCALL_SETUP();
 	fctx = (directory_fctx *) funcctx->user_fctx;
 
-	while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL)
+	if (populate_paths(fctx, funcctx))
 	{
-		Datum		values[3];
-		bool		nulls[3];
-		char		path[MAXPGPATH * 2];
-		struct stat attrib;
+		Datum		values[4];
+		bool		nulls[4] = {0};
 		HeapTuple	tuple;
 
-		/* Skip hidden files */
-		if (de->d_name[0] == '.')
-			continue;
-
-		/* Get the file info */
-		snprintf(path, sizeof(path), "%s/%s", fctx->location, de->d_name);
-		if (stat(path, &attrib) < 0)
-			ereport(ERROR,
-					(errcode_for_file_access(),
-					 errmsg("could not stat file \"%s\": %m", path)));
-
-		/* Ignore anything but regular files */
-		if (!S_ISREG(attrib.st_mode))
-			continue;
-
-		values[0] = CStringGetTextDatum(de->d_name);
-		values[1] = Int64GetDatum((int64) attrib.st_size);
-		values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime));
-		memset(nulls, 0, sizeof(nulls));
+		values[0] = CStringGetTextDatum(fctx->path);
+		values[1] = Int64GetDatum((int64) fctx->stat.st_size);
+		values[2] = TimestampTzGetDatum(time_t_to_timestamptz(fctx->stat.st_mtime));
+		if (dir_action != DIR_HIDE)
+			values[3] = BoolGetDatum(S_ISDIR(fctx->stat.st_mode));
 
 		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		pfree(fctx->path);
 		SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple));
 	}
 
-	FreeDir(fctx->dirdesc);
+	FreeDir(linitial(fctx->dirdesc));
+	list_free(fctx->dirdesc);
+	list_free_deep(fctx->location);
 	SRF_RETURN_DONE(funcctx);
 }
 
@@ -613,14 +690,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 Datum
 pg_ls_logdir(PG_FUNCTION_ARGS)
 {
-	return pg_ls_dir_files(fcinfo, Log_directory, false);
+	return pg_ls_dir_files(fcinfo, Log_directory, false, DIR_HIDE);
 }
 
 /* Function to return the list of files in the WAL directory */
 Datum
 pg_ls_waldir(PG_FUNCTION_ARGS)
 {
-	return pg_ls_dir_files(fcinfo, XLOGDIR, false);
+	return pg_ls_dir_files(fcinfo, XLOGDIR, false, DIR_HIDE);
 }
 
 /*
@@ -638,7 +715,7 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc)
 						tblspc)));
 
 	TempTablespacePath(path, tblspc);
-	return pg_ls_dir_files(fcinfo, path, true);
+	return pg_ls_dir_files(fcinfo, path, true, DIR_DESCEND);
 }
 
 /*
@@ -667,5 +744,5 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS)
 Datum
 pg_ls_archive_statusdir(PG_FUNCTION_ARGS)
 {
-	return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true);
+	return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true, DIR_HIDE);
 }
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 07a86c7b7b..0dbce4ba09 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10739,13 +10739,13 @@
 { oid => '5029', descr => 'list files in the pgsql_tmp directory',
   proname => 'pg_ls_tmpdir', procost => '10', prorows => '20', proretset => 't',
   provolatile => 'v', prorettype => 'record', proargtypes => '',
-  proallargtypes => '{text,int8,timestamptz}', proargmodes => '{o,o,o}',
-  proargnames => '{name,size,modification}', prosrc => 'pg_ls_tmpdir_noargs' },
+  proallargtypes => '{text,int8,timestamptz,bool}', proargmodes => '{o,o,o,o}',
+  proargnames => '{name,size,modification,isdir}', prosrc => 'pg_ls_tmpdir_noargs' },
 { oid => '5030', descr => 'list files in the pgsql_tmp directory',
   proname => 'pg_ls_tmpdir', procost => '10', prorows => '20', proretset => 't',
   provolatile => 'v', prorettype => 'record', proargtypes => 'oid',
-  proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}',
-  proargnames => '{tablespace,name,size,modification}',
+  proallargtypes => '{oid,text,int8,timestamptz,bool}', proargmodes => '{i,o,o,o,o}',
+  proargnames => '{tablespace,name,size,modification,isdir}',
   prosrc => 'pg_ls_tmpdir_1arg' },
 
 # hash partitioning constraint function
diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source
index a5f61a35dc..a7f34c5dd5 100644
--- a/src/test/regress/input/tablespace.source
+++ b/src/test/regress/input/tablespace.source
@@ -11,6 +11,11 @@ DROP TABLESPACE regress_tblspacewith;
 -- create a tablespace we can use
 CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@';
 
+-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet
+-- The name='' condition is never true, so the function runs to completion but returns zero rows.
+-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir()
+SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1)b , pg_ls_tmpdir(oid)c WHERE c.name='Does not exist';
+
 -- try setting and resetting some properties for the new tablespace
 ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1);
 ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true);  -- fail
diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source
index 162b591b31..edcdb4f32a 100644
--- a/src/test/regress/output/tablespace.source
+++ b/src/test/regress/output/tablespace.source
@@ -13,6 +13,14 @@ SELECT spcoptions FROM pg_tablespace WHERE spcname = 'regress_tblspacewith';
 DROP TABLESPACE regress_tblspacewith;
 -- create a tablespace we can use
 CREATE TABLESPACE regress_tblspace LOCATION '@testtablespace@';
+-- This tests the missing_ok parameter, which causes pg_ls_tmpdir to succeed even if the tmpdir doesn't exist yet
+-- The name='' condition is never true, so the function runs to completion but returns zero rows.
+-- The query is written to ERROR if the tablespace doesn't exist, rather than silently failing to call pg_ls_tmpdir()
+SELECT c.* FROM (SELECT oid FROM pg_tablespace b WHERE b.spcname='regress_tblspace' UNION SELECT 0 ORDER BY 1 DESC LIMIT 1)b , pg_ls_tmpdir(oid)c WHERE c.name='Does not exist';
+ name | size | modification | isdir 
+------+------+--------------+-------
+(0 rows)
+
 -- try setting and resetting some properties for the new tablespace
 ALTER TABLESPACE regress_tblspace SET (random_page_cost = 1.0, seq_page_cost = 1.1);
 ALTER TABLESPACE regress_tblspace SET (some_nonexistent_parameter = true);  -- fail
-- 
2.17.0

