Re: BUG #17254: Crash with 0xC0000409 in pg_stat_statements when pg_stat_tmp\pgss_query_texts.stat exceeded 2GB. - Mailing list pgsql-bugs

I wrote:
> =?UTF-8?Q?Juan_Jos=C3=A9_Santamar=C3=ADa_Flecha?= <juanjo.santamaria@gmail.com> writes:
>> On Sat, Oct 30, 2021 at 6:26 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>>> I think instead, we need to turn the subsequent one-off read() call into a
>>> loop that reads no more than INT_MAX bytes at a time.  It'd be possible
>>> to restrict that to Windows, but probably no harm in doing it the same
>>> way everywhere.

>> Seems reasonable to me, can such a change be back-patched?

> Don't see why not.

Here's a quick patch for that.  I don't have any ability to check it
on Windows, but the logic is easy to verify by reducing the arbitrary
constant to something small.  (I used 1GB, not INT_MAX, because I figured
we ought to read in multiples of a filesystem block if possible.)

            regards, tom lane

diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c
index 07fe0e7cda..726ba59e2b 100644
--- a/contrib/pg_stat_statements/pg_stat_statements.c
+++ b/contrib/pg_stat_statements/pg_stat_statements.c
@@ -2125,6 +2125,7 @@ qtext_load_file(Size *buffer_size)
     char       *buf;
     int            fd;
     struct stat stat;
+    Size        nread;

     fd = OpenTransientFile(PGSS_TEXT_FILE, O_RDONLY | PG_BINARY);
     if (fd < 0)
@@ -2165,23 +2166,35 @@ qtext_load_file(Size *buffer_size)
     }

     /*
-     * OK, slurp in the file.  If we get a short read and errno doesn't get
-     * set, the reason is probably that garbage collection truncated the file
-     * since we did the fstat(), so we don't log a complaint --- but we don't
-     * return the data, either, since it's most likely corrupt due to
-     * concurrent writes from garbage collection.
+     * OK, slurp in the file.  Windows fails if we try to read more than
+     * INT_MAX bytes at once, and other platforms might not like that either,
+     * so read a very large file in 1GB segments.
      */
-    errno = 0;
-    if (read(fd, buf, stat.st_size) != stat.st_size)
+    nread = 0;
+    while (nread < stat.st_size)
     {
-        if (errno)
-            ereport(LOG,
-                    (errcode_for_file_access(),
-                     errmsg("could not read file \"%s\": %m",
-                            PGSS_TEXT_FILE)));
-        free(buf);
-        CloseTransientFile(fd);
-        return NULL;
+        int            toread = Min(1024 * 1024 * 1024, stat.st_size - nread);
+
+        /*
+         * If we get a short read and errno doesn't get set, the reason is
+         * probably that garbage collection truncated the file since we did
+         * the fstat(), so we don't log a complaint --- but we don't return
+         * the data, either, since it's most likely corrupt due to concurrent
+         * writes from garbage collection.
+         */
+        errno = 0;
+        if (read(fd, buf + nread, toread) != toread)
+        {
+            if (errno)
+                ereport(LOG,
+                        (errcode_for_file_access(),
+                         errmsg("could not read file \"%s\": %m",
+                                PGSS_TEXT_FILE)));
+            free(buf);
+            CloseTransientFile(fd);
+            return NULL;
+        }
+        nread += toread;
     }

     if (CloseTransientFile(fd) != 0)
@@ -2189,7 +2202,7 @@ qtext_load_file(Size *buffer_size)
                 (errcode_for_file_access(),
                  errmsg("could not close file \"%s\": %m", PGSS_TEXT_FILE)));

-    *buffer_size = stat.st_size;
+    *buffer_size = nread;
     return buf;
 }