From 95ed868179a09ffe374316d0356a128adcebfe71 Mon Sep 17 00:00:00 2001 From: Jacob Champion Date: Mon, 3 May 2021 15:38:26 -0700 Subject: [PATCH v22 1/4] common/jsonapi: support libpq as a client Based on a patch by Michael Paquier. For frontend code, use PQExpBuffer instead of StringInfo. This requires us to track allocation failures so that we can return JSON_OUT_OF_MEMORY as needed rather than exit()ing. --- src/bin/pg_combinebackup/Makefile | 4 +- src/bin/pg_combinebackup/meson.build | 2 +- src/bin/pg_verifybackup/Makefile | 2 +- src/common/Makefile | 2 +- src/common/jsonapi.c | 173 ++++++++++++++++++++------- src/common/meson.build | 8 +- src/include/common/jsonapi.h | 19 ++- 7 files changed, 158 insertions(+), 52 deletions(-) diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile index c3729755ba..2f7dc1ed87 100644 --- a/src/bin/pg_combinebackup/Makefile +++ b/src/bin/pg_combinebackup/Makefile @@ -18,6 +18,8 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils +# TODO: fix this properly +LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport) OBJS = \ $(WIN32RES) \ @@ -30,7 +32,7 @@ OBJS = \ all: pg_combinebackup -pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils +pg_combinebackup: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) install: all installdirs diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build index 1d4b9c218f..cab677b574 100644 --- a/src/bin/pg_combinebackup/meson.build +++ b/src/bin/pg_combinebackup/meson.build @@ -17,7 +17,7 @@ endif pg_combinebackup = executable('pg_combinebackup', pg_combinebackup_sources, - dependencies: [frontend_code], + dependencies: [frontend_code, libpq], kwargs: default_bin_args, ) bin_targets += pg_combinebackup diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile index 7c045f142e..3372fada01 100644 --- a/src/bin/pg_verifybackup/Makefile +++ b/src/bin/pg_verifybackup/Makefile @@ -17,7 +17,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global # We need libpq only because fe_utils does. -LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) +LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpgcommon $(libpq_pgport) OBJS = \ $(WIN32RES) \ diff --git a/src/common/Makefile b/src/common/Makefile index 3d83299432..c4f4448e2f 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -40,7 +40,7 @@ override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(LDFLAGS_SL)\"" override CPPFLAGS += -DVAL_LIBS="\"$(LIBS)\"" -override CPPFLAGS := -DFRONTEND -I. -I$(top_srcdir)/src/common $(CPPFLAGS) +override CPPFLAGS := -DFRONTEND -I. -I$(top_srcdir)/src/common -I$(libpq_srcdir) $(CPPFLAGS) LIBS += $(PTHREAD_LIBS) OBJS_COMMON = \ diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c index 98d6e66a21..4aeedc0bc5 100644 --- a/src/common/jsonapi.c +++ b/src/common/jsonapi.c @@ -21,10 +21,43 @@ #include "mb/pg_wchar.h" #include "port/pg_lfind.h" -#ifndef FRONTEND +#ifdef FRONTEND +#include "pqexpbuffer.h" +#else +#include "lib/stringinfo.h" #include "miscadmin.h" #endif +/* + * In backend, we will use palloc/pfree along with StringInfo. In frontend, use + * malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory. + */ +#ifdef FRONTEND + +#define STRDUP(s) strdup(s) +#define ALLOC(size) malloc(size) + +#define appendStrVal appendPQExpBuffer +#define appendBinaryStrVal appendBinaryPQExpBuffer +#define appendStrValChar appendPQExpBufferChar +#define createStrVal createPQExpBuffer +#define resetStrVal resetPQExpBuffer +#define destroyStrVal destroyPQExpBuffer + +#else /* !FRONTEND */ + +#define STRDUP(s) pstrdup(s) +#define ALLOC(size) palloc(size) + +#define appendStrVal appendStringInfo +#define appendBinaryStrVal appendBinaryStringInfo +#define appendStrValChar appendStringInfoChar +#define createStrVal makeStringInfo +#define resetStrVal resetStringInfo +#define destroyStrVal destroyStringInfo + +#endif + /* * The context of the parser is maintained by the recursive descent * mechanism, but is passed explicitly to the error reporting routine @@ -168,9 +201,16 @@ makeJsonLexContextCstringLen(JsonLexContext *lex, char *json, lex->input_encoding = encoding; if (need_escapes) { - lex->strval = makeStringInfo(); + /* + * This call can fail in FRONTEND code. We defer error handling to + * time of use (json_lex_string()) since we might not need to parse + * any strings anyway. + */ + lex->strval = createStrVal(); lex->flags |= JSONLEX_FREE_STRVAL; + lex->parse_strval = true; } + lex->errormsg = NULL; return lex; } @@ -185,14 +225,18 @@ makeJsonLexContextCstringLen(JsonLexContext *lex, char *json, void freeJsonLexContext(JsonLexContext *lex) { + static const JsonLexContext empty = {0}; + if (lex->flags & JSONLEX_FREE_STRVAL) - destroyStringInfo(lex->strval); + destroyStrVal(lex->strval); if (lex->errormsg) - destroyStringInfo(lex->errormsg); + destroyStrVal(lex->errormsg); if (lex->flags & JSONLEX_FREE_STRUCT) pfree(lex); + else + *lex = empty; } /* @@ -258,7 +302,7 @@ json_count_array_elements(JsonLexContext *lex, int *elements) * etc, so doing this with a copy makes that safe. */ memcpy(©lex, lex, sizeof(JsonLexContext)); - copylex.strval = NULL; /* not interested in values here */ + copylex.parse_strval = false; /* not interested in values here */ copylex.lex_level++; count = 0; @@ -320,14 +364,21 @@ parse_scalar(JsonLexContext *lex, JsonSemAction *sem) /* extract the de-escaped string value, or the raw lexeme */ if (lex_peek(lex) == JSON_TOKEN_STRING) { - if (lex->strval != NULL) - val = pstrdup(lex->strval->data); + if (lex->parse_strval) + { + val = STRDUP(lex->strval->data); + if (val == NULL) + return JSON_OUT_OF_MEMORY; + } } else { int len = (lex->token_terminator - lex->token_start); - val = palloc(len + 1); + val = ALLOC(len + 1); + if (val == NULL) + return JSON_OUT_OF_MEMORY; + memcpy(val, lex->token_start, len); val[len] = '\0'; } @@ -361,8 +412,12 @@ parse_object_field(JsonLexContext *lex, JsonSemAction *sem) if (lex_peek(lex) != JSON_TOKEN_STRING) return report_parse_error(JSON_PARSE_STRING, lex); - if ((ostart != NULL || oend != NULL) && lex->strval != NULL) - fname = pstrdup(lex->strval->data); + if ((ostart != NULL || oend != NULL) && lex->parse_strval) + { + fname = STRDUP(lex->strval->data); + if (fname == NULL) + return JSON_OUT_OF_MEMORY; + } result = json_lex(lex); if (result != JSON_SUCCESS) return result; @@ -418,6 +473,11 @@ parse_object(JsonLexContext *lex, JsonSemAction *sem) JsonParseErrorType result; #ifndef FRONTEND + + /* + * TODO: clients need some way to put a bound on stack growth. Parse level + * limits maybe? + */ check_stack_depth(); #endif @@ -766,8 +826,15 @@ json_lex_string(JsonLexContext *lex) return code; \ } while (0) - if (lex->strval != NULL) - resetStringInfo(lex->strval); + if (lex->parse_strval) + { +#ifdef FRONTEND + /* make sure initialization succeeded */ + if (lex->strval == NULL) + return JSON_OUT_OF_MEMORY; +#endif + resetStrVal(lex->strval); + } Assert(lex->input_length > 0); s = lex->token_start; @@ -804,7 +871,7 @@ json_lex_string(JsonLexContext *lex) else FAIL_AT_CHAR_END(JSON_UNICODE_ESCAPE_FORMAT); } - if (lex->strval != NULL) + if (lex->parse_strval) { /* * Combine surrogate pairs. @@ -861,19 +928,19 @@ json_lex_string(JsonLexContext *lex) unicode_to_utf8(ch, (unsigned char *) utf8str); utf8len = pg_utf_mblen((unsigned char *) utf8str); - appendBinaryStringInfo(lex->strval, utf8str, utf8len); + appendBinaryPQExpBuffer(lex->strval, utf8str, utf8len); } else if (ch <= 0x007f) { /* The ASCII range is the same in all encodings */ - appendStringInfoChar(lex->strval, (char) ch); + appendPQExpBufferChar(lex->strval, (char) ch); } else FAIL_AT_CHAR_END(JSON_UNICODE_HIGH_ESCAPE); #endif /* FRONTEND */ } } - else if (lex->strval != NULL) + else if (lex->parse_strval) { if (hi_surrogate != -1) FAIL_AT_CHAR_END(JSON_UNICODE_LOW_SURROGATE); @@ -883,22 +950,22 @@ json_lex_string(JsonLexContext *lex) case '"': case '\\': case '/': - appendStringInfoChar(lex->strval, *s); + appendStrValChar(lex->strval, *s); break; case 'b': - appendStringInfoChar(lex->strval, '\b'); + appendStrValChar(lex->strval, '\b'); break; case 'f': - appendStringInfoChar(lex->strval, '\f'); + appendStrValChar(lex->strval, '\f'); break; case 'n': - appendStringInfoChar(lex->strval, '\n'); + appendStrValChar(lex->strval, '\n'); break; case 'r': - appendStringInfoChar(lex->strval, '\r'); + appendStrValChar(lex->strval, '\r'); break; case 't': - appendStringInfoChar(lex->strval, '\t'); + appendStrValChar(lex->strval, '\t'); break; default: @@ -933,7 +1000,7 @@ json_lex_string(JsonLexContext *lex) /* * Skip to the first byte that requires special handling, so we - * can batch calls to appendBinaryStringInfo. + * can batch calls to appendBinaryStrVal. */ while (p < end - sizeof(Vector8) && !pg_lfind8('\\', (uint8 *) p, sizeof(Vector8)) && @@ -957,8 +1024,8 @@ json_lex_string(JsonLexContext *lex) } } - if (lex->strval != NULL) - appendBinaryStringInfo(lex->strval, s, p - s); + if (lex->parse_strval) + appendBinaryStrVal(lex->strval, s, p - s); /* * s will be incremented at the top of the loop, so set it to just @@ -974,6 +1041,11 @@ json_lex_string(JsonLexContext *lex) return JSON_UNICODE_LOW_SURROGATE; } +#ifdef FRONTEND + if (lex->parse_strval && PQExpBufferBroken(lex->strval)) + return JSON_OUT_OF_MEMORY; +#endif + /* Hooray, we found the end of the string! */ lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; @@ -1158,19 +1230,25 @@ report_parse_error(JsonParseContext ctx, JsonLexContext *lex) char * json_errdetail(JsonParseErrorType error, JsonLexContext *lex) { + if (error == JSON_OUT_OF_MEMORY) + { + /* Short circuit. Allocating anything for this case is unhelpful. */ + return _("out of memory"); + } + if (lex->errormsg) - resetStringInfo(lex->errormsg); + resetStrVal(lex->errormsg); else - lex->errormsg = makeStringInfo(); + lex->errormsg = createStrVal(); /* * A helper for error messages that should print the current token. The * format must contain exactly one %.*s specifier. */ #define token_error(lex, format) \ - appendStringInfo((lex)->errormsg, _(format), \ - (int) ((lex)->token_terminator - (lex)->token_start), \ - (lex)->token_start); + appendStrVal((lex)->errormsg, _(format), \ + (int) ((lex)->token_terminator - (lex)->token_start), \ + (lex)->token_start); switch (error) { @@ -1181,9 +1259,9 @@ json_errdetail(JsonParseErrorType error, JsonLexContext *lex) token_error(lex, "Escape sequence \"\\%.*s\" is invalid."); break; case JSON_ESCAPING_REQUIRED: - appendStringInfo(lex->errormsg, - _("Character with value 0x%02x must be escaped."), - (unsigned char) *(lex->token_terminator)); + appendStrVal(lex->errormsg, + _("Character with value 0x%02x must be escaped."), + (unsigned char) *(lex->token_terminator)); break; case JSON_EXPECTED_END: token_error(lex, "Expected end of input, but found \"%.*s\"."); @@ -1214,6 +1292,9 @@ json_errdetail(JsonParseErrorType error, JsonLexContext *lex) case JSON_INVALID_TOKEN: token_error(lex, "Token \"%.*s\" is invalid."); break; + case JSON_OUT_OF_MEMORY: + /* should have been handled above; use the error path */ + break; case JSON_UNICODE_CODE_POINT_ZERO: return _("\\u0000 cannot be converted to text."); case JSON_UNICODE_ESCAPE_FORMAT: @@ -1245,15 +1326,23 @@ json_errdetail(JsonParseErrorType error, JsonLexContext *lex) } #undef token_error - /* - * We don't use a default: case, so that the compiler will warn about - * unhandled enum values. But this needs to be here anyway to cover the - * possibility of an incorrect input. - */ - if (lex->errormsg->len == 0) - appendStringInfo(lex->errormsg, - _("unexpected json parse error type: %d"), - (int) error); + /* Note that lex->errormsg can be NULL in FRONTEND code. */ + if (lex->errormsg && lex->errormsg->len == 0) + { + /* + * We don't use a default: case, so that the compiler will warn about + * unhandled enum values. But this needs to be here anyway to cover + * the possibility of an incorrect input. + */ + appendStrVal(lex->errormsg, + _("unexpected json parse error type: %d"), + (int) error); + } + +#ifdef FRONTEND + if (PQExpBufferBroken(lex->errormsg)) + return _("out of memory while constructing error description"); +#endif return lex->errormsg->data; } diff --git a/src/common/meson.build b/src/common/meson.build index de68e408fa..379b228d86 100644 --- a/src/common/meson.build +++ b/src/common/meson.build @@ -125,13 +125,18 @@ common_sources_frontend_static += files( # least cryptohash_openssl.c, hmac_openssl.c depend on it. # controldata_utils.c depends on wait_event_types_h. That's arguably a # layering violation, but ... +# +# XXX Frontend builds need libpq's pqexpbuffer.h, so adjust the include paths +# appropriately. This seems completely broken. pgcommon = {} pgcommon_variants = { '_srv': internal_lib_args + { + 'include_directories': include_directories('.'), 'sources': common_sources + [lwlocknames_h] + [wait_event_types_h], 'dependencies': [backend_common_code], }, '': default_lib_args + { + 'include_directories': include_directories('../interfaces/libpq', '.'), 'sources': common_sources_frontend_static, 'dependencies': [frontend_common_code], # Files in libpgcommon.a should use/export the "xxx_private" versions @@ -140,6 +145,7 @@ pgcommon_variants = { }, '_shlib': default_lib_args + { 'pic': true, + 'include_directories': include_directories('../interfaces/libpq', '.'), 'sources': common_sources_frontend_shlib, 'dependencies': [frontend_common_code], }, @@ -157,7 +163,6 @@ foreach name, opts : pgcommon_variants c_args = opts.get('c_args', []) + common_cflags[cflagname] cflag_libs += static_library('libpgcommon@0@_@1@'.format(name, cflagname), c_pch: pch_c_h, - include_directories: include_directories('.'), kwargs: opts + { 'sources': sources, 'c_args': c_args, @@ -170,7 +175,6 @@ foreach name, opts : pgcommon_variants lib = static_library('libpgcommon@0@'.format(name), link_with: cflag_libs, c_pch: pch_c_h, - include_directories: include_directories('.'), kwargs: opts + { 'dependencies': opts['dependencies'] + [ssl], } diff --git a/src/include/common/jsonapi.h b/src/include/common/jsonapi.h index 86a0fc2d00..75d444c17a 100644 --- a/src/include/common/jsonapi.h +++ b/src/include/common/jsonapi.h @@ -14,8 +14,6 @@ #ifndef JSONAPI_H #define JSONAPI_H -#include "lib/stringinfo.h" - typedef enum JsonTokenType { JSON_TOKEN_INVALID, @@ -48,6 +46,7 @@ typedef enum JsonParseErrorType JSON_EXPECTED_OBJECT_NEXT, JSON_EXPECTED_STRING, JSON_INVALID_TOKEN, + JSON_OUT_OF_MEMORY, JSON_UNICODE_CODE_POINT_ZERO, JSON_UNICODE_ESCAPE_FORMAT, JSON_UNICODE_HIGH_ESCAPE, @@ -57,6 +56,17 @@ typedef enum JsonParseErrorType JSON_SEM_ACTION_FAILED, /* error should already be reported */ } JsonParseErrorType; +/* + * Don't depend on the internal type header for strval; if callers need access + * then they can include the appropriate header themselves. + */ +#ifdef FRONTEND +#define StrValType PQExpBufferData +#else +#define StrValType StringInfoData +#endif + +typedef struct StrValType StrValType; /* * All the fields in this structure should be treated as read-only. @@ -88,8 +98,9 @@ typedef struct JsonLexContext bits32 flags; int line_number; /* line number, starting from 1 */ char *line_start; /* where that line starts within input */ - StringInfo strval; - StringInfo errormsg; + bool parse_strval; + StrValType *strval; /* only used if parse_strval == true */ + StrValType *errormsg; } JsonLexContext; typedef JsonParseErrorType (*json_struct_action) (void *state); -- 2.32.1 (Apple Git-133)