Re: pgbench - allow backslash-continuations in custom scripts - Mailing list pgsql-hackers

So I looked into this, and found that persuading psql to let backslash
commands cross line boundaries is a much bigger deal than just fixing the
lexer.  The problem is that MainLoop would need to grow an understanding
of having received only a partial backslash command and needing to go back
to readline() for another line.  And probably HandleSlashCmds would need
to be changed to stop parsing and back out without doing anything when it
hits backslash-newline.  It's do-able no doubt, but it's not going to be a
small and simple patch.

However, since pgbench is already set up to slurp the entire file and
lex it in one go, it is just a trivial adjustment to the lexer rules in
that program.  The only thing I found that made it complicated is that
syntax_error() had too simplistic an understanding of how to position
the error cursor usefully, so that needed a bit of work.

I think it'd be okay to commit this without fixing psql at the same time;
if you try it in psql you get an error, so on that side it's unimplemented
behavior rather than an actual incompatibility.  Perhaps somebody will be
motivated to fix it later, but I'm not going to spend that kind of time
on it right now.

I've not written a docs update, but otherwise I think this is committable.
Comments?

            regards, tom lane

diff --git a/src/bin/pgbench/exprscan.l b/src/bin/pgbench/exprscan.l
index 825dacc..12b5f7e 100644
*** a/src/bin/pgbench/exprscan.l
--- b/src/bin/pgbench/exprscan.l
***************
*** 6,17 ****
   *
   * This lexer supports two operating modes:
   *
!  * In INITIAL state, just parse off whitespace-separated words (this mode
!  * is basically equivalent to strtok(), which is what we used to use).
   *
   * In EXPR state, lex for the simple expression syntax of exprparse.y.
   *
!  * In either mode, stop upon hitting newline or end of string.
   *
   * Note that this lexer operates within the framework created by psqlscan.l,
   *
--- 6,18 ----
   *
   * This lexer supports two operating modes:
   *
!  * In INITIAL state, just parse off whitespace-separated words.  (This mode
!  * is basically equivalent to strtok(), which is what we used to use.)
   *
   * In EXPR state, lex for the simple expression syntax of exprparse.y.
   *
!  * In either mode, stop upon hitting newline, end of string, or unquoted
!  * backslash (except that backslash-newline is silently swallowed).
   *
   * Note that this lexer operates within the framework created by psqlscan.l,
   *
*************** extern void expr_yyset_column(int column
*** 61,69 ****
  alpha            [a-zA-Z_]
  digit            [0-9]
  alnum            [a-zA-Z0-9_]
! /* {space} + {nonspace} + {newline} should cover all characters */
  space            [ \t\r\f\v]
! nonspace        [^ \t\r\f\v\n]
  newline            [\n]

  /* Exclusive states */
--- 62,71 ----
  alpha            [a-zA-Z_]
  digit            [0-9]
  alnum            [a-zA-Z0-9_]
! /* {space} + {nonspace} + {backslash} + {newline} should cover all characters */
  space            [ \t\r\f\v]
! nonspace        [^ \t\r\f\v\\\n]
! backslash        [\\]
  newline            [\n]

  /* Exclusive states */
*************** newline            [\n]
*** 98,103 ****
--- 100,113 ----

  {space}+        { /* ignore */ }

+ {backslash}{newline}    { /* ignore */ }
+
+ {backslash}        {
+                     /* do not eat, and report end of command */
+                     yyless(0);
+                     return 0;
+                 }
+
  {newline}        {
                      /* report end of command */
                      last_was_newline = true;
*************** newline            [\n]
*** 130,143 ****
                      return FUNCTION;
                  }

  {newline}        {
                      /* report end of command */
                      last_was_newline = true;
                      return 0;
                  }

- {space}+        { /* ignore */ }
-
  .                {
                      /*
                       * must strdup yytext so that expr_yyerror_more doesn't
--- 140,161 ----
                      return FUNCTION;
                  }

+ {space}+        { /* ignore */ }
+
+ {backslash}{newline}    { /* ignore */ }
+
+ {backslash}        {
+                     /* do not eat, and report end of command */
+                     yyless(0);
+                     return 0;
+                 }
+
  {newline}        {
                      /* report end of command */
                      last_was_newline = true;
                      return 0;
                  }

  .                {
                      /*
                       * must strdup yytext so that expr_yyerror_more doesn't
*************** expr_yyerror_more(yyscan_t yyscanner, co
*** 177,183 ****
      /*
       * While parsing an expression, we may not have collected the whole line
       * yet from the input source.  Lex till EOL so we can report whole line.
!      * (If we're at EOF, it's okay to call yylex() an extra time.)
       */
      if (!last_was_newline)
      {
--- 195,201 ----
      /*
       * While parsing an expression, we may not have collected the whole line
       * yet from the input source.  Lex till EOL so we can report whole line.
!      * (If we're at backslash/EOF, it's okay to call yylex() an extra time.)
       */
      if (!last_was_newline)
      {
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 4196b0e..e947f77 100644
*** a/src/bin/pgbench/pgbench.c
--- b/src/bin/pgbench/pgbench.c
*************** syntax_error(const char *source, int lin
*** 2413,2426 ****
      fprintf(stderr, "\n");
      if (line != NULL)
      {
!         fprintf(stderr, "%s\n", line);
!         if (column >= 0)
          {
              int            i;

!             for (i = 0; i < column; i++)
!                 fprintf(stderr, " ");
!             fprintf(stderr, "^ error found here\n");
          }
      }
      exit(1);
--- 2413,2455 ----
      fprintf(stderr, "\n");
      if (line != NULL)
      {
!         /*
!          * Multi-line backslash commands make this harder than you'd think; we
!          * have to identify which line to put the error cursor on.  So print
!          * one line at a time.
!          */
!         for (;;)
          {
+             const char *nlpos = strchr(line, '\n');
+             int            len;
              int            i;

!             if (nlpos)
!             {
!                 /*
!                  * It's tempting to use fprintf("%.*s"), but that can fail if
!                  * glibc has a different idea of the encoding than we do.
!                  */
!                 len = nlpos - line + 1;
!                 for (i = 0; i < len; i++)
!                     fputc(line[i], stderr);
!             }
!             else
!             {
!                 len = column + 1;        /* ensure we print ^ if not done */
!                 fprintf(stderr, "%s\n", line);
!             }
!             if (column >= 0 && column < len)
!             {
!                 for (i = 0; i < column; i++)
!                     fputc(' ', stderr);
!                 fprintf(stderr, "^ error found here\n");
!             }
!             column -= len;
!             if (nlpos)
!                 line = nlpos + 1;
!             else
!                 break;
          }
      }
      exit(1);