From 802b5bde48cc378dc69baa1e781548bb7182fb45 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Wed, 6 Mar 2024 13:17:41 +0000 Subject: [PATCH v4] Add detailed info when COPY skips soft errors --- doc/src/sgml/ref/copy.sgml | 21 +++++++++++++++++++-- src/backend/commands/copy.c | 8 ++++++++ src/backend/commands/copyfromparse.c | 10 ++++++++++ src/bin/psql/tab-complete.c | 2 +- src/include/commands/copy.h | 1 + src/test/regress/expected/copy2.out | 6 +++++- src/test/regress/sql/copy2.sql | 2 +- 7 files changed, 45 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 55764fc1f2..d0e58c0f9f 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -44,6 +44,7 @@ COPY { table_name [ ( column_name [, ...] ) | * } FORCE_NULL { ( column_name [, ...] ) | * } ON_ERROR 'error_action' + LOG_VERBOSITY [ boolean ] ENCODING 'encoding_name' @@ -397,8 +398,12 @@ COPY { table_name [ ( FORMAT is text or csv. - A NOTICE message containing the ignored row count is emitted at the end - of the COPY FROM if at least one row was discarded. + A NOTICE message containing the ignored row count is + emitted at the end of the COPY FROM if at least one + row was discarded. When LOG_VERBOSITY option is set to + true (or equivalent Boolean value), a + NOTICE message containing the line number and column + name for each discarded row is emitted. @@ -415,6 +420,18 @@ COPY { table_name [ ( + + LOG_VERBOSITY + + + Sets the verbosity of logged messages by COPY + command. As an example, see its usage for + COPY FROM command's ON_ERROR + clause with ignore option. + + + + WHERE diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 056b6733c8..aa9fee5a71 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -454,6 +454,7 @@ ProcessCopyOptions(ParseState *pstate, bool freeze_specified = false; bool header_specified = false; bool on_error_specified = false; + bool log_verbosity_specified = false; ListCell *option; /* Support external use for option sanity checking */ @@ -613,6 +614,13 @@ ProcessCopyOptions(ParseState *pstate, on_error_specified = true; opts_out->on_error = defGetCopyOnErrorChoice(defel, pstate, is_from); } + else if (strcmp(defel->defname, "log_verbosity") == 0) + { + if (log_verbosity_specified) + errorConflictingDefElem(defel, pstate); + log_verbosity_specified = true; + opts_out->log_verbosity = defGetBoolean(defel); + } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 5682d5d054..5f6be5c400 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -967,7 +967,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, (Node *) cstate->escontext, &values[m])) { + Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); + cstate->num_errors++; + + if (cstate->opts.log_verbosity) + ereport(NOTICE, + errmsg("detected data type incompatibility at line number %llu for column %s; COPY %s", + (unsigned long long) cstate->cur_lineno, + cstate->cur_attname, + cstate->cur_relname)); + return true; } diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index aa1acf8523..b6d5767acc 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2900,7 +2900,7 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("FORMAT", "FREEZE", "DELIMITER", "NULL", "HEADER", "QUOTE", "ESCAPE", "FORCE_QUOTE", "FORCE_NOT_NULL", "FORCE_NULL", "ENCODING", "DEFAULT", - "ON_ERROR"); + "ON_ERROR", "LOG_VERBOSITY"); /* Complete COPY FROM|TO filename WITH (FORMAT */ else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "FORMAT")) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index b3da3cb0be..e194081fad 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -73,6 +73,7 @@ typedef struct CopyFormatOptions bool *force_null_flags; /* per-column CSV FN flags */ bool convert_selectively; /* do selective binary conversion? */ CopyOnErrorChoice on_error; /* what to do when error happened */ + bool log_verbosity; /* log more verbose messages? */ List *convert_select; /* list of column names (can be NIL) */ } CopyFormatOptions; diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 25c401ce34..07e52bcd4a 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -729,7 +729,11 @@ CREATE TABLE check_ign_err (n int, m int[], k int); COPY check_ign_err FROM STDIN WITH (on_error stop); ERROR: invalid input syntax for type integer: "a" CONTEXT: COPY check_ign_err, line 2, column n: "a" -COPY check_ign_err FROM STDIN WITH (on_error ignore); +COPY check_ign_err FROM STDIN WITH (on_error ignore, log_verbosity on); +NOTICE: detected data type incompatibility at line number 2 for column n; COPY check_ign_err +NOTICE: detected data type incompatibility at line number 3 for column k; COPY check_ign_err +NOTICE: detected data type incompatibility at line number 4 for column m; COPY check_ign_err +NOTICE: detected data type incompatibility at line number 5 for column n; COPY check_ign_err NOTICE: 4 rows were skipped due to data type incompatibility SELECT * FROM check_ign_err; n | m | k diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index b5e549e856..47d131c1ce 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -508,7 +508,7 @@ a {2} 2 5 {5} 5 \. -COPY check_ign_err FROM STDIN WITH (on_error ignore); +COPY check_ign_err FROM STDIN WITH (on_error ignore, log_verbosity on); 1 {1} 1 a {2} 2 3 {3} 3333333333 -- 2.34.1