diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 033eae9b46..e0e65f9c21 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -661,9 +661,9 @@ PostgreSQL documentation
...). This will make restoration very slow; it is mainly
useful for making dumps that can be loaded into
non-PostgreSQL databases.
- However, since this option generates a separate command for each row,
- an error in reloading a row causes only that row to be lost rather
- than the entire table contents.
+ Any error during reloading will cause only rows that are part of the
+ problematic INSERT to be lost, rather than the
+ entire table contents.
@@ -775,13 +775,12 @@ PostgreSQL documentation
than COPY). This will make restoration very slow;
it is mainly useful for making dumps that can be loaded into
non-PostgreSQL databases.
- However, since this option generates a separate command for each row,
- an error in reloading a row causes only that row to be lost rather
- than the entire table contents.
- Note that
- the restore might fail altogether if you have rearranged column order.
- The option is safe against column
- order changes, though even slower.
+ Any error during reloading will cause only rows that are part of the
+ problematic INSERT to be lost, rather than the
+ entire table contents. Note that the restore might fail altogether if
+ you have rearranged column order. The
+ option is safe against column order
+ changes, though even slower.
@@ -925,8 +924,9 @@ PostgreSQL documentation
Add ON CONFLICT DO NOTHING to
INSERT commands.
- This option is not valid unless or
- is also specified.
+ This option is not valid unless ,
+ or
+ is also specified.
@@ -949,6 +949,20 @@ PostgreSQL documentation
+
+
+
+
+ Dump data as INSERT commands (rather than
+ COPY). Controls the maximum number of rows per
+ INSERT command. The value specified must be a
+ number greater than zero. Any error during reloading will cause only
+ rows that are part of the problematic INSERT to be
+ lost, rather than the entire table contents.
+
+
+
+
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 4a2e122e2d..7ab27391fb 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -140,10 +140,10 @@ typedef struct _dumpOptions
int dumpSections; /* bitmask of chosen sections */
bool aclsSkip;
const char *lockWaitTimeout;
+ int dump_inserts; /* 0 = COPY, otherwise rows per INSERT */
/* flags for various command-line long options */
int disable_dollar_quoting;
- int dump_inserts;
int column_inserts;
int if_exists;
int no_comments;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 5d83038348..9d6e25aee1 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -311,6 +311,8 @@ main(int argc, char **argv)
const char *dumpencoding = NULL;
const char *dumpsnapshot = NULL;
char *use_role = NULL;
+ char *rowPerInsertEndPtr;
+ long rowPerInsert;
int numWorkers = 1;
trivalue prompt_password = TRI_DEFAULT;
int compressLevel = -1;
@@ -363,7 +365,7 @@ main(int argc, char **argv)
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
{"if-exists", no_argument, &dopt.if_exists, 1},
- {"inserts", no_argument, &dopt.dump_inserts, 1},
+ {"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
{"no-tablespaces", no_argument, &dopt.outputNoTablespaces, 1},
{"quote-all-identifiers", no_argument, "e_all_identifiers, 1},
@@ -382,6 +384,7 @@ main(int argc, char **argv)
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-sync", no_argument, NULL, 7},
{"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1},
+ {"rows-per-insert", required_argument, NULL, 10},
{NULL, 0, NULL, 0}
};
@@ -572,6 +575,29 @@ main(int argc, char **argv)
}
break;
+ case 9: /* inserts */
+ /*
+ * dump_inserts also stores --rows-per-insert, careful not to
+ * overwrite that.
+ */
+ if (dopt.dump_inserts == 0)
+ dopt.dump_inserts = DUMP_DEFAULT_ROWS_PER_INSERT;
+ break;
+
+ case 10: /* rows per insert */
+ errno = 0;
+ rowPerInsert = strtol(optarg, &rowPerInsertEndPtr, 10);
+
+ if (rowPerInsertEndPtr == optarg || *rowPerInsertEndPtr != '\0' ||
+ rowPerInsert > INT_MAX || rowPerInsert <= 0 || errno == ERANGE)
+ {
+ write_msg(NULL, "rows-per-insert must be in range %d..%d\n",
+ 1, INT_MAX);
+ exit_nicely(1);
+ }
+ dopt.dump_inserts = rowPerInsert;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit_nicely(1);
@@ -596,8 +622,8 @@ main(int argc, char **argv)
}
/* --column-inserts implies --inserts */
- if (dopt.column_inserts)
- dopt.dump_inserts = 1;
+ if (dopt.column_inserts && dopt.dump_inserts == 0)
+ dopt.dump_inserts = DUMP_DEFAULT_ROWS_PER_INSERT;
/*
* Binary upgrade mode implies dumping sequence data even in schema-only
@@ -622,8 +648,12 @@ main(int argc, char **argv)
if (dopt.if_exists && !dopt.outputClean)
exit_horribly(NULL, "option --if-exists requires option -c/--clean\n");
- if (dopt.do_nothing && !(dopt.dump_inserts || dopt.column_inserts))
- exit_horribly(NULL, "option --on-conflict-do-nothing requires option --inserts or --column-inserts\n");
+ /*
+ * --inserts are already implied above if --column-inserts or
+ * --rows-per-insert were specified.
+ */
+ if (dopt.do_nothing && dopt.dump_inserts == 0)
+ exit_horribly(NULL, "option --on-conflict-do-nothing requires option --inserts , --rows-per-insert or --column-inserts\n");
/* Identify archive format to emit */
archiveFormat = parseArchiveFormat(format, &archiveMode);
@@ -993,6 +1023,7 @@ help(const char *progname)
printf(_(" --no-unlogged-table-data do not dump unlogged table data\n"));
printf(_(" --on-conflict-do-nothing add ON CONFLICT DO NOTHING to INSERT commands\n"));
printf(_(" --quote-all-identifiers quote all identifiers, even if not key words\n"));
+ printf(_(" --rows-per-insert=NROWS number of rows per INSERT command\n"));
printf(_(" --section=SECTION dump named section (pre-data, data, or post-data)\n"));
printf(_(" --serializable-deferrable wait until the dump can run without anomalies\n"));
printf(_(" --snapshot=SNAPSHOT use given snapshot for the dump\n"));
@@ -1912,6 +1943,8 @@ dumpTableData_insert(Archive *fout, void *dcontext)
int tuple;
int nfields;
int field;
+ int rows_per_statement = dopt->dump_inserts;
+ int rows_this_statement = 0;
appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR "
"SELECT * FROM ONLY %s",
@@ -1926,68 +1959,88 @@ dumpTableData_insert(Archive *fout, void *dcontext)
res = ExecuteSqlQuery(fout, "FETCH 100 FROM _pg_dump_cursor",
PGRES_TUPLES_OK);
nfields = PQnfields(res);
- for (tuple = 0; tuple < PQntuples(res); tuple++)
+
+ /*
+ * First time through, we build as much of the INSERT statement as
+ * possible in "insertStmt", which we can then just print for each
+ * line. If the table happens to have zero columns then this will
+ * be a complete statement, otherwise it will end in "VALUES" and
+ * be ready to have the row's column values printed.
+ */
+ if (insertStmt == NULL)
{
- /*
- * First time through, we build as much of the INSERT statement as
- * possible in "insertStmt", which we can then just print for each
- * line. If the table happens to have zero columns then this will
- * be a complete statement, otherwise it will end in "VALUES(" and
- * be ready to have the row's column values appended.
- */
- if (insertStmt == NULL)
- {
- TableInfo *targettab;
+ TableInfo *targettab;
- insertStmt = createPQExpBuffer();
+ insertStmt = createPQExpBuffer();
- /*
- * When load-via-partition-root is set, get the root table
- * name for the partition table, so that we can reload data
- * through the root table.
- */
- if (dopt->load_via_partition_root && tbinfo->ispartition)
- targettab = getRootTableInfo(tbinfo);
- else
- targettab = tbinfo;
+ /*
+ * When load-via-partition-root is set, get the root table
+ * name for the partition table, so that we can reload data
+ * through the root table.
+ */
+ if (dopt->load_via_partition_root && tbinfo->ispartition)
+ targettab = getRootTableInfo(tbinfo);
+ else
+ targettab = tbinfo;
- appendPQExpBuffer(insertStmt, "INSERT INTO %s ",
- fmtQualifiedDumpable(targettab));
+ appendPQExpBuffer(insertStmt, "INSERT INTO %s ",
+ fmtQualifiedDumpable(targettab));
- /* corner case for zero-column table */
- if (nfields == 0)
- {
- appendPQExpBufferStr(insertStmt, "DEFAULT VALUES;\n");
- }
- else
+ /* corner case for zero-column table */
+ if (nfields == 0)
+ {
+ appendPQExpBufferStr(insertStmt, "DEFAULT VALUES;\n");
+ }
+ else
+ {
+ /* append the list of column names if required */
+ if (dopt->column_inserts)
{
- /* append the list of column names if required */
- if (dopt->column_inserts)
+ appendPQExpBufferChar(insertStmt, '(');
+ for (field = 0; field < nfields; field++)
{
- appendPQExpBufferChar(insertStmt, '(');
- for (field = 0; field < nfields; field++)
- {
- if (field > 0)
- appendPQExpBufferStr(insertStmt, ", ");
- appendPQExpBufferStr(insertStmt,
- fmtId(PQfname(res, field)));
- }
- appendPQExpBufferStr(insertStmt, ") ");
+ if (field > 0)
+ appendPQExpBufferStr(insertStmt, ", ");
+ appendPQExpBufferStr(insertStmt,
+ fmtId(PQfname(res, field)));
}
+ appendPQExpBufferStr(insertStmt, ") ");
+ }
- if (tbinfo->needs_override)
- appendPQExpBufferStr(insertStmt, "OVERRIDING SYSTEM VALUE ");
+ if (tbinfo->needs_override)
+ appendPQExpBufferStr(insertStmt, "OVERRIDING SYSTEM VALUE ");
- appendPQExpBufferStr(insertStmt, "VALUES (");
- }
+ appendPQExpBufferStr(insertStmt, "VALUES");
}
+ }
- archputs(insertStmt->data, fout);
+ for (tuple = 0; tuple < PQntuples(res); tuple++)
+ {
+ /* Write the INSERT if not in the middle of a multi-row INSERT. */
+ if (rows_this_statement == 0)
+ archputs(insertStmt->data, fout);
- /* if it is zero-column table then we're done */
+
+ /*
+ * If it is zero-column table then we've aleady written the
+ * complete statement, which will mean we've disobeyed
+ * --rows-per-insert when it's set greater than 1. We do support
+ * a way to make this multi-row with:
+ * SELECT UNION ALL SELECT UNION ALL ... but that's non-standard
+ * so likely we should avoid it given that using INSERTs is
+ * mostly only ever needed for cross-database exports.
+ */
if (nfields == 0)
continue;
+ if (rows_this_statement > 0)
+ archputs(",\n\t(", fout);
+ else if (rows_per_statement == 1)
+ archputs(" (", fout);
+ else
+ archputs("\n\t(", fout);
+
+
for (field = 0; field < nfields; field++)
{
if (field > 0)
@@ -2053,10 +2106,27 @@ dumpTableData_insert(Archive *fout, void *dcontext)
}
}
- if (!dopt->do_nothing)
- archputs(");\n", fout);
+ rows_this_statement++;
+
+ /*
+ * If we've put the target number of rows onto this statement then
+ * we can terminate it now.
+ */
+ if (rows_this_statement == rows_per_statement)
+ {
+ /* Reset the row counter */
+ rows_this_statement = 0;
+ if (dopt->do_nothing)
+ archputs(") ON CONFLICT DO NOTHING;\n", fout);
+ else
+ archputs(");\n", fout);
+ }
else
- archputs(") ON CONFLICT DO NOTHING;\n", fout);
+ {
+ /* Otherwise, get ready for the next row. */
+ archputs(")", fout);
+ }
+
}
if (PQntuples(res) <= 0)
@@ -2067,6 +2137,15 @@ dumpTableData_insert(Archive *fout, void *dcontext)
PQclear(res);
}
+ /* Terminate any statements that didn't make the row count.*/
+ if (rows_this_statement > 0)
+ {
+ if (dopt->do_nothing)
+ archputs(" ON CONFLICT DO NOTHING;\n", fout);
+ else
+ archputs(";\n", fout);
+ }
+
archputs("\n\n", fout);
ExecuteSqlStatement(fout, "CLOSE _pg_dump_cursor");
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 21d2ab05b0..59ac3d096e 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -126,6 +126,12 @@ typedef uint32 DumpComponents; /* a bitmask of dump object components */
DUMP_COMPONENT_DATA |\
DUMP_COMPONENT_POLICY)
+/*
+ * The default number of rows per INSERT statement when
+ * --inserts is specified without --rows-per-insert
+ */
+#define DUMP_DEFAULT_ROWS_PER_INSERT 1
+
typedef struct _dumpableObject
{
DumpableObjectType objType;
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 1dad553739..45dd57f2bf 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -118,8 +118,8 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--on-conflict-do-nothing' ],
- qr/\Qpg_dump: option --on-conflict-do-nothing requires option --inserts or --column-inserts\E/,
- 'pg_dump: option --on-conflict-do-nothing requires option --inserts or --column-inserts');
+ qr/\Qpg_dump: option --on-conflict-do-nothing requires option --inserts , --rows-per-insert or --column-inserts\E/,
+ 'pg_dump: option --on-conflict-do-nothing requires option --inserts , --rows-per-insert or --column-inserts');
# pg_dumpall command-line argument checks
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 8fa7f0f61f..c5a8f763af 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -295,6 +295,26 @@ my %pgdump_runs = (
"$tempdir/role_parallel",
],
},
+ rows_per_insert => {
+ dump_cmd => [
+ 'pg_dump',
+ '--no-sync',
+ "--file=$tempdir/rows_per_insert.sql", '-a',
+ '--rows-per-insert=3',
+ '--table=dump_test.test_table',
+ 'postgres',
+ ],
+ },
+ rows_per_insert_zero_col => {
+ dump_cmd => [
+ 'pg_dump',
+ '--no-sync',
+ "--file=$tempdir/rows_per_insert_zero_col.sql", '-a',
+ '--rows-per-insert=3',
+ '--table=dump_test.dump_test.test_fourth_table',
+ 'postgres',
+ ],
+ },
schema_only => {
dump_cmd => [
'pg_dump', '--format=plain',
@@ -1295,6 +1315,13 @@ my %tests = (
like => { column_inserts => 1, },
},
+ 'INSERT INTO test_table' => {
+ regexp => qr/^
+ (?:INSERT\ INTO\ dump_test\.test_table\ VALUES\n\t\(\d,\ NULL,\ NULL,\ NULL\),\n\t\(\d,\ NULL,\ NULL,\ NULL\),\n\t\(\d,\ NULL,\ NULL,\ NULL\);\n){3}
+ /xm,
+ like => { rows_per_insert => 1, },
+ },
+
'INSERT INTO test_second_table' => {
regexp => qr/^
(?:INSERT\ INTO\ dump_test\.test_second_table\ \(col1,\ col2\)
@@ -1305,7 +1332,7 @@ my %tests = (
'INSERT INTO test_fourth_table' => {
regexp =>
qr/^\QINSERT INTO dump_test.test_fourth_table DEFAULT VALUES;\E/m,
- like => { column_inserts => 1, },
+ like => { column_inserts => 1, rows_per_insert_zero_col => 1 },
},
'INSERT INTO test_fifth_table' => {