From d3109cf275f7dc28b96b1738f51754b4078af573 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki.linnakangas@iki.fi>
Date: Sat, 8 Jul 2023 22:19:48 +0300
Subject: [PATCH v8 5/7] Determine array dimensions and parse the elements in
 one pass.

The logic in ArrayCount() and ReadArrayStr() was hard to follow,
because they mixed low-level escaping and quoting with tracking the
curly braces. Introduce a ReadArrayToken() function that tokenizes the
input, handling quoting and escaping.

Instead of having separate ArrayCount() and ReadArrayStr() functions, move
the logic for determining and checking the structure of the dimensions
to ReadArrayStr(). ReadArrayStr() now determines the dimensions and
parses the elements in one pass.

ReadArrayStr() no longer scribbles on the input. Instead, it allocates
a separate scratch buffer that it uses to hold the string representation
of each element. The scratch buffer must be as large as the input string,
so this doesn't save memory, but palloc(strlen(x)) is cheaper than
pstrdup(x).
---
 src/backend/utils/adt/arrayfuncs.c   | 1044 +++++++++++---------------
 src/test/regress/expected/arrays.out |    2 +-
 src/tools/pgindent/typedefs.list     |    2 +-
 3 files changed, 440 insertions(+), 608 deletions(-)

diff --git a/src/backend/utils/adt/arrayfuncs.c b/src/backend/utils/adt/arrayfuncs.c
index 7a63db09..a2d806e3 100644
--- a/src/backend/utils/adt/arrayfuncs.c
+++ b/src/backend/utils/adt/arrayfuncs.c
@@ -53,18 +53,6 @@ bool		Array_nulls = true;
 			PG_FREE_IF_COPY(array, n); \
 	} while (0)
 
-typedef enum
-{
-	ARRAY_NO_LEVEL,
-	ARRAY_LEVEL_STARTED,
-	ARRAY_ELEM_STARTED,
-	ARRAY_QUOTED_ELEM_STARTED,
-	ARRAY_QUOTED_ELEM_COMPLETED,
-	ARRAY_ELEM_DELIMITED,
-	ARRAY_LEVEL_COMPLETED,
-	ARRAY_LEVEL_DELIMITED
-} ArrayParseState;
-
 /* Working state for array_iterate() */
 typedef struct ArrayIteratorData
 {
@@ -89,18 +77,30 @@ typedef struct ArrayIteratorData
 	int			current_item;	/* the item # we're at in the array */
 }			ArrayIteratorData;
 
+/* ReadArrayToken return type */
+typedef enum
+{
+	ATOK_LEVEL_START,
+	ATOK_LEVEL_END,
+	ATOK_DELIM,
+	ATOK_ELEM,
+	ATOK_ELEM_NULL,
+	ATOK_ERROR,
+} ArrayToken;
+
 static bool ReadDimensionInt(char **srcptr, int *result, const char *origStr, Node *escontext);
 static bool ReadArrayDimensions(char **srcptr, int *ndim, int *dim, int *lBound,
 								const char *origStr, Node *escontext);
-static int	ArrayCount(const char *str, int *dim, char typdelim,
-					   Node *escontext);
-static bool ReadArrayStr(char *arrayStr, const char *origStr,
-						 int nitems,
+static bool ReadArrayStr(char **srcptr,
 						 FmgrInfo *inputproc, Oid typioparam, int32 typmod,
 						 char typdelim,
 						 int typlen, bool typbyval, char typalign,
-						 Datum *values, bool *nulls,
-						 bool *hasnulls, int32 *nbytes, Node *escontext);
+						 int *ndim, int *dim,
+						 int *nitems,
+						 Datum **values, bool **nulls,
+						 const char *origStr, Node *escontext);
+static ArrayToken ReadArrayToken(char **srcptr, char *elembuf, char typdelim,
+								 const char *origStr, Node *escontext);
 static void ReadArrayBinary(StringInfo buf, int nitems,
 							FmgrInfo *receiveproc, Oid typioparam, int32 typmod,
 							int typlen, bool typbyval, char typalign,
@@ -186,12 +186,11 @@ array_in(PG_FUNCTION_ARGS)
 	char		typalign;
 	char		typdelim;
 	Oid			typioparam;
-	char	   *string_save,
-			   *p;
-	int			i,
-				nitems;
-	Datum	   *dataPtr;
-	bool	   *nullsPtr;
+	char	   *p;
+	int			nitems;
+	int			nitems_according_to_dims;
+	Datum	   *values;
+	bool	   *nulls;
 	bool		hasnulls;
 	int32		nbytes;
 	int32		dataoffset;
@@ -234,15 +233,13 @@ array_in(PG_FUNCTION_ARGS)
 	typdelim = my_extra->typdelim;
 	typioparam = my_extra->typioparam;
 
-	/* Make a modifiable copy of the input */
-	string_save = pstrdup(string);
-
 	/*
+	 * Start processing the input string.
+	 *
 	 * If the input string starts with dimension info, read and use that.
-	 * Otherwise, we require the input to be in curly-brace style, and we
-	 * prescan the input to determine dimensions.
+	 * Otherwise, we determine the dimensions from the curly braces.
 	 */
-	p = string_save;
+	p = string;
 	if (!ReadArrayDimensions(&p, &ndim, dim, lBound, string, escontext))
 		return (Datum) 0;
 
@@ -254,17 +251,11 @@ array_in(PG_FUNCTION_ARGS)
 					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
 					 errmsg("malformed array literal: \"%s\"", string),
 					 errdetail("Array value must start with \"{\" or dimension information.")));
-		ndim = ArrayCount(p, dim, typdelim, escontext);
-		if (ndim < 0)
-			PG_RETURN_NULL();
-		for (i = 0; i < ndim; i++)
+		for (int i = 0; i < MAXDIM; i++)
 			lBound[i] = 1;
 	}
 	else
 	{
-		int			ndim_braces,
-					dim_braces[MAXDIM];
-
 		/* If array dimensions are given, expect '=' operator */
 		if (strncmp(p, ASSGN, strlen(ASSGN)) != 0)
 			ereturn(escontext, (Datum) 0,
@@ -276,46 +267,62 @@ array_in(PG_FUNCTION_ARGS)
 		while (scanner_isspace(*p))
 			p++;
 
-		/*
-		 * intuit dimensions from brace structure -- it better match what we
-		 * were given
-		 */
 		if (*p != '{')
 			ereturn(escontext, (Datum) 0,
 					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
 					 errmsg("malformed array literal: \"%s\"", string),
 					 errdetail("Array contents must start with \"{\".")));
-		ndim_braces = ArrayCount(p, dim_braces, typdelim, escontext);
-		if (ndim_braces < 0)
-			PG_RETURN_NULL();
-		if (ndim_braces != ndim)
-			ereturn(escontext, (Datum) 0,
+	}
+
+	/* Parse the value part, in the curly braces: { ... } */
+	if (!ReadArrayStr(&p,
+					  &my_extra->proc, typioparam, typmod,
+					  typdelim,
+					  typlen, typbyval, typalign,
+					  &ndim,
+					  dim,
+					  &nitems,
+					  &values, &nulls,
+					  string,
+					  escontext))
+		PG_RETURN_NULL();
+
+	/* only whitespace is allowed after the closing brace */
+	while (*p)
+	{
+		if (!scanner_isspace(*p++))
+			ereturn(escontext, false,
 					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
 					 errmsg("malformed array literal: \"%s\"", string),
-					 errdetail("Specified array dimensions do not match array contents.")));
-		for (i = 0; i < ndim; ++i)
-		{
-			if (dim[i] != dim_braces[i])
-				ereturn(escontext, (Datum) 0,
-						(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-						 errmsg("malformed array literal: \"%s\"", string),
-						 errdetail("Specified array dimensions do not match array contents.")));
-		}
+					 errdetail("Junk after closing right brace.")));
 	}
 
 #ifdef ARRAYDEBUG
-	printf("array_in- ndim %d (", ndim);
-	for (i = 0; i < ndim; i++)
 	{
-		printf(" %d", dim[i]);
-	};
-	printf(") for %s\n", string);
+		StringInfoData buf;
+
+		initStringInfo(&buf);
+
+		appendStringInfo(&buf, "array_in- ndim %d (", ndim);
+		for (int i = 0; i < ndim; i++)
+			appendStringInfo(&buf, " %d", dim[i]);
+		appendStringInfo(&buf, ") for %s\n", string);
+		elog(NOTICE, "%s", buf.data);
+		pfree(buf.data);
+	}
 #endif
 
 	/* This checks for overflow of the array dimensions */
-	nitems = ArrayGetNItemsSafe(ndim, dim, escontext);
-	if (nitems < 0)
+
+	/*
+	 * FIXME: Is this still required? I believe all the checks it performs are
+	 * redundant with other checks in ReadArrayDimension() and ReadArrayStr()
+	 */
+	nitems_according_to_dims = ArrayGetNItemsSafe(ndim, dim, escontext);
+	if (nitems_according_to_dims < 0)
 		PG_RETURN_NULL();
+	if (nitems != nitems_according_to_dims)
+		elog(ERROR, "mismatch nitems, %d vs %d", nitems, nitems_according_to_dims);
 	if (!ArrayCheckBoundsSafe(ndim, dim, lBound, escontext))
 		PG_RETURN_NULL();
 
@@ -323,16 +330,30 @@ array_in(PG_FUNCTION_ARGS)
 	if (nitems == 0)
 		PG_RETURN_ARRAYTYPE_P(construct_empty_array(element_type));
 
-	dataPtr = (Datum *) palloc(nitems * sizeof(Datum));
-	nullsPtr = (bool *) palloc(nitems * sizeof(bool));
-	if (!ReadArrayStr(p, string,
-					  nitems,
-					  &my_extra->proc, typioparam, typmod,
-					  typdelim,
-					  typlen, typbyval, typalign,
-					  dataPtr, nullsPtr,
-					  &hasnulls, &nbytes, escontext))
-		PG_RETURN_NULL();
+	/*
+	 * Check for nulls, compute total data space needed
+	 */
+	hasnulls = false;
+	nbytes = 0;
+	for (int i = 0; i < nitems; i++)
+	{
+		if (nulls[i])
+			hasnulls = true;
+		else
+		{
+			/* let's just make sure data is not toasted */
+			if (typlen == -1)
+				values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i]));
+			nbytes = att_addlength_datum(nbytes, typlen, values[i]);
+			nbytes = att_align_nominal(nbytes, typalign);
+			/* check for overflow of total request */
+			if (!AllocSizeIsValid(nbytes))
+				ereturn(escontext, false,
+						(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+						 errmsg("array size exceeds the maximum allowed (%d)",
+								(int) MaxAllocSize)));
+		}
+	}
 	if (hasnulls)
 	{
 		dataoffset = ARR_OVERHEAD_WITHNULLS(ndim, nitems);
@@ -343,6 +364,10 @@ array_in(PG_FUNCTION_ARGS)
 		dataoffset = 0;			/* marker for no null bitmap */
 		nbytes += ARR_OVERHEAD_NONULLS(ndim);
 	}
+
+	/*
+	 * Construct the final array datum
+	 */
 	retval = (ArrayType *) palloc0(nbytes);
 	SET_VARSIZE(retval, nbytes);
 	retval->ndim = ndim;
@@ -358,18 +383,16 @@ array_in(PG_FUNCTION_ARGS)
 	memcpy(ARR_LBOUND(retval), lBound, ndim * sizeof(int));
 
 	CopyArrayEls(retval,
-				 dataPtr, nullsPtr, nitems,
+				 values, nulls, nitems,
 				 typlen, typbyval, typalign,
 				 true);
 
-	pfree(dataPtr);
-	pfree(nullsPtr);
-	pfree(string_save);
+	pfree(values);
+	pfree(nulls);
 
 	PG_RETURN_ARRAYTYPE_P(retval);
 }
 
-
 /*
  * ReadArrayDimensions
  *	 parses the array dimensions part from "src" and converts the values
@@ -380,8 +403,7 @@ array_in(PG_FUNCTION_ARGS)
  *
  * *ndim_p, *dim, and *lBound are output variables. They are filled with the
  * number of dimensions (<= MAXDIM), the length of each dimension, and the
- * lower bounds of the slices, respectively. If there is no dimension
- * information, *ndim_p is set to 0.
+ * lower bounds of the slices, respectively.
  *
  * 'origStr' is the original input string, used only in error messages.
  * If *escontext points to an ErrorSaveContext, details of any error are
@@ -493,7 +515,7 @@ ReadArrayDimensions(char **srcptr, int *ndim_p, int *dim, int *lBound,
  *
  * Result:
  *	true for success, false for failure (if escontext is provided).
- *	On success, the parsed integer is returned in *result.
+ *  On success, the parsed integer is returned in *result.
  */
 static bool
 ReadDimensionInt(char **srcptr, int *result, const char *origStr, Node *escontext)
@@ -516,350 +538,40 @@ ReadDimensionInt(char **srcptr, int *result, const char *origStr, Node *escontex
 	return true;
 }
 
-/*
- * ArrayCount
- *	 Determines the dimensions for an array string.  This includes
- *	 syntax-checking the array structure decoration (braces and delimiters).
- *
- * Returns number of dimensions as function result.  The axis lengths are
- * returned in dim[], which must be of size MAXDIM.  This function does not
- * special-case empty arrays: it will return a positive result with some
- * zero axis length(s).
- *
- * If we detect an error, fill *escontext with error details and return -1
- * (unless escontext isn't provided, in which case errors will be thrown).
- */
-static int
-ArrayCount(const char *str, int *dim, char typdelim, Node *escontext)
-{
-	int			nest_level = 0,
-				i;
-	int			ndim = 1,
-				nelems[MAXDIM];
-	bool		ndim_frozen = false;
-	bool		in_quotes = false;
-	bool		eoArray = false;
-	const char *ptr;
-	ArrayParseState parse_state = ARRAY_NO_LEVEL;
-
-	/* Initialize dim[] entries to -1 meaning "unknown" */
-	for (i = 0; i < MAXDIM; ++i)
-		dim[i] = -1;
-
-	/* Scan string until we reach closing brace */
-	ptr = str;
-	while (!eoArray)
-	{
-		bool		new_element = false;
-
-		switch (*ptr)
-		{
-			case '\0':
-				/* Signal a premature end of the string */
-				ereturn(escontext, -1,
-						(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-						 errmsg("malformed array literal: \"%s\"", str),
-						 errdetail("Unexpected end of input.")));
-			case '\\':
-
-				/*
-				 * An escape must be after a level start, after an element
-				 * start, or after an element delimiter. In any case we now
-				 * must be past an element start.
-				 */
-				switch (parse_state)
-				{
-					case ARRAY_LEVEL_STARTED:
-					case ARRAY_ELEM_DELIMITED:
-						/* start new unquoted element */
-						parse_state = ARRAY_ELEM_STARTED;
-						new_element = true;
-						break;
-					case ARRAY_ELEM_STARTED:
-					case ARRAY_QUOTED_ELEM_STARTED:
-						/* already in element */
-						break;
-					default:
-						ereturn(escontext, -1,
-								(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-								 errmsg("malformed array literal: \"%s\"", str),
-								 errdetail("Unexpected \"%c\" character.",
-										   '\\')));
-				}
-				/* skip the escaped character */
-				if (*(ptr + 1))
-					ptr++;
-				else
-					ereturn(escontext, -1,
-							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-							 errmsg("malformed array literal: \"%s\"", str),
-							 errdetail("Unexpected end of input.")));
-				break;
-			case '"':
-
-				/*
-				 * A quote must be after a level start, after a quoted element
-				 * start, or after an element delimiter. In any case we now
-				 * must be past an element start.
-				 */
-				switch (parse_state)
-				{
-					case ARRAY_LEVEL_STARTED:
-					case ARRAY_ELEM_DELIMITED:
-						/* start new quoted element */
-						Assert(!in_quotes);
-						in_quotes = true;
-						parse_state = ARRAY_QUOTED_ELEM_STARTED;
-						new_element = true;
-						break;
-					case ARRAY_QUOTED_ELEM_STARTED:
-						/* already in element, end it */
-						Assert(in_quotes);
-						in_quotes = false;
-						parse_state = ARRAY_QUOTED_ELEM_COMPLETED;
-						break;
-					default:
-						ereturn(escontext, -1,
-								(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-								 errmsg("malformed array literal: \"%s\"", str),
-								 errdetail("Unexpected array element.")));
-				}
-				break;
-			case '{':
-				if (!in_quotes)
-				{
-					/*
-					 * A left brace can occur if no nesting has occurred yet,
-					 * after a level start, or after a level delimiter.  If we
-					 * see one after an element delimiter, we have something
-					 * like "{1,{2}}", so complain about non-rectangularity.
-					 */
-					switch (parse_state)
-					{
-						case ARRAY_NO_LEVEL:
-						case ARRAY_LEVEL_STARTED:
-						case ARRAY_LEVEL_DELIMITED:
-							/* okay */
-							break;
-						case ARRAY_ELEM_DELIMITED:
-							ereturn(escontext, -1,
-									(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-									 errmsg("malformed array literal: \"%s\"", str),
-									 errdetail("Multidimensional arrays must have sub-arrays with matching dimensions.")));
-						default:
-							ereturn(escontext, -1,
-									(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-									 errmsg("malformed array literal: \"%s\"", str),
-									 errdetail("Unexpected \"%c\" character.",
-											   '{')));
-					}
-					parse_state = ARRAY_LEVEL_STARTED;
-					/* Nested sub-arrays count as elements of outer level */
-					if (nest_level > 0)
-						nelems[nest_level - 1]++;
-					/* Initialize element counting in the new level */
-					if (nest_level >= MAXDIM)
-						ereturn(escontext, -1,
-								(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
-								 errmsg("number of array dimensions (%d) exceeds the maximum allowed (%d)",
-										nest_level + 1, MAXDIM)));
-					nelems[nest_level] = 0;
-					nest_level++;
-					if (ndim < nest_level)
-					{
-						/* Can't increase ndim once it's frozen */
-						if (ndim_frozen)
-							ereturn(escontext, -1,
-									(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-									 errmsg("malformed array literal: \"%s\"", str),
-									 errdetail("Multidimensional arrays must have sub-arrays with matching dimensions.")));
-						ndim = nest_level;
-					}
-				}
-				break;
-			case '}':
-				if (!in_quotes)
-				{
-					/*
-					 * A right brace can occur after an element start, an
-					 * element completion, a quoted element completion, or a
-					 * level completion.  We also allow it after a level
-					 * start, that is an empty sub-array "{}" --- but that
-					 * freezes the number of dimensions and all such
-					 * sub-arrays must be at the same level, just like
-					 * sub-arrays containing elements.
-					 */
-					switch (parse_state)
-					{
-						case ARRAY_ELEM_STARTED:
-						case ARRAY_QUOTED_ELEM_COMPLETED:
-						case ARRAY_LEVEL_COMPLETED:
-							/* okay */
-							break;
-						case ARRAY_LEVEL_STARTED:
-							/* empty sub-array: OK if at correct nest_level */
-							ndim_frozen = true;
-							if (nest_level != ndim)
-								ereturn(escontext, -1,
-										(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-										 errmsg("malformed array literal: \"%s\"", str),
-										 errdetail("Multidimensional arrays must have sub-arrays with matching dimensions.")));
-							break;
-						default:
-							ereturn(escontext, -1,
-									(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-									 errmsg("malformed array literal: \"%s\"", str),
-									 errdetail("Unexpected \"%c\" character.",
-											   '}')));
-					}
-					parse_state = ARRAY_LEVEL_COMPLETED;
-					/* The parse state check assured we're in a level. */
-					Assert(nest_level > 0);
-					nest_level--;
-
-					if (dim[nest_level] < 0)
-					{
-						/* Save length of first sub-array of this level */
-						dim[nest_level] = nelems[nest_level];
-					}
-					else if (nelems[nest_level] != dim[nest_level])
-					{
-						/* Subsequent sub-arrays must have same length */
-						ereturn(escontext, -1,
-								(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-								 errmsg("malformed array literal: \"%s\"", str),
-								 errdetail("Multidimensional arrays must have sub-arrays with matching dimensions.")));
-					}
-					/* Done if this is the outermost level's '}' */
-					if (nest_level == 0)
-						eoArray = true;
-				}
-				break;
-			default:
-				if (!in_quotes)
-				{
-					if (*ptr == typdelim)
-					{
-						/*
-						 * Delimiters can occur after an element start, a
-						 * quoted element completion, or a level completion.
-						 */
-						if (parse_state != ARRAY_ELEM_STARTED &&
-							parse_state != ARRAY_QUOTED_ELEM_COMPLETED &&
-							parse_state != ARRAY_LEVEL_COMPLETED)
-							ereturn(escontext, -1,
-									(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-									 errmsg("malformed array literal: \"%s\"", str),
-									 errdetail("Unexpected \"%c\" character.",
-											   typdelim)));
-						if (parse_state == ARRAY_LEVEL_COMPLETED)
-							parse_state = ARRAY_LEVEL_DELIMITED;
-						else
-							parse_state = ARRAY_ELEM_DELIMITED;
-					}
-					else if (!scanner_isspace(*ptr))
-					{
-						/*
-						 * Other non-space characters must be after a level
-						 * start, after an element start, or after an element
-						 * delimiter. In any case we now must be past an
-						 * element start.
-						 *
-						 * If it's a space character, we can ignore it; it
-						 * might be data or not, but it doesn't change the
-						 * parsing state.
-						 */
-						switch (parse_state)
-						{
-							case ARRAY_LEVEL_STARTED:
-							case ARRAY_ELEM_DELIMITED:
-								/* start new unquoted element */
-								parse_state = ARRAY_ELEM_STARTED;
-								new_element = true;
-								break;
-							case ARRAY_ELEM_STARTED:
-								/* already in element */
-								break;
-							default:
-								ereturn(escontext, -1,
-										(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-										 errmsg("malformed array literal: \"%s\"", str),
-										 errdetail("Unexpected array element.")));
-						}
-					}
-				}
-				break;
-		}
-
-		/* To reduce duplication, all new-element cases go through here. */
-		if (new_element)
-		{
-			/*
-			 * Once we have found an element, the number of dimensions can no
-			 * longer increase, and subsequent elements must all be at the
-			 * same nesting depth.
-			 */
-			ndim_frozen = true;
-			if (nest_level != ndim)
-				ereturn(escontext, -1,
-						(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-						 errmsg("malformed array literal: \"%s\"", str),
-						 errdetail("Multidimensional arrays must have sub-arrays with matching dimensions.")));
-			/* Count the new element */
-			nelems[nest_level - 1]++;
-		}
-
-		ptr++;
-	}
-
-	/* only whitespace is allowed after the closing brace */
-	while (*ptr)
-	{
-		if (!scanner_isspace(*ptr++))
-			ereturn(escontext, -1,
-					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-					 errmsg("malformed array literal: \"%s\"", str),
-					 errdetail("Junk after closing right brace.")));
-	}
-
-	return ndim;
-}
-
 /*
  * ReadArrayStr :
- *	 parses the array string pointed to by "arrayStr" and converts the values
- *	 to internal format.  The array dimensions must have been determined,
- *	 and the case of an empty array must have been handled earlier.
+ *	 parses the array string pointed to by *srcptr and converts the values
+ *	 to internal format.  Determines the array dimensions as it goes.
  *
- * Inputs:
- *	arrayStr: the string to parse.
- *			  CAUTION: the contents of "arrayStr" will be modified!
- *	origStr: the unmodified input string, used only in error messages.
- *	nitems: total number of array elements, as already determined.
+ * On entry, *srcptr points to the string to parse. On successful return,
+ * it is advanced to point past the closing '}'.
+ *
+ * If dimensions were specified explicitly, they are passed in *ndim_p and
+ * dim. This function will check that the array values match the specified
+ * dimensions. If *ndim_p == 0, this function deduces the dimensions from
+ * the structure of the input, and stores them in *ndim_p and the dim array.
+ *
+ * Element type information:
  *	inputproc: type-specific input procedure for element datatype.
  *	typioparam, typmod: auxiliary values to pass to inputproc.
  *	typdelim: the value delimiter (type-specific).
  *	typlen, typbyval, typalign: storage parameters of element datatype.
  *
  * Outputs:
- *	values[]: filled with converted data values.
- *	nulls[]: filled with is-null markers.
- *	*hasnulls: set true iff there are any null elements.
- *	*nbytes: set to total size of data area needed (including alignment
- *		padding but not including array header overhead).
- *	*escontext: if this points to an ErrorSaveContext, details of
- *		any error are reported there.
+ *  *ndim_p, dim: dimensions deduced from the input structure
+ *  *nitems_p: total number of elements
+ *	*values[]: palloc'd array, filled with converted data values.
+ *	*nulls[]: palloc'd array, filled with is-null markers.
+ *
+ * 'origStr' is the original input string, used only in error messages.
+ * If *escontext points to an ErrorSaveContext, details of any error are
+ * reported there.
  *
  * Result:
  *	true for success, false for failure (if escontext is provided).
- *
- * Note that values[] and nulls[] are allocated by the caller, and must have
- * nitems elements.
  */
 static bool
-ReadArrayStr(char *arrayStr,
-			 const char *origStr,
-			 int nitems,
+ReadArrayStr(char **srcptr,
 			 FmgrInfo *inputproc,
 			 Oid typioparam,
 			 int32 typmod,
@@ -867,225 +579,345 @@ ReadArrayStr(char *arrayStr,
 			 int typlen,
 			 bool typbyval,
 			 char typalign,
-			 Datum *values,
-			 bool *nulls,
-			 bool *hasnulls,
-			 int32 *nbytes,
+			 int *ndim_p,
+			 int *dim,
+			 int *nitems_p,
+			 Datum **values_p,
+			 bool **nulls_p,
+			 const char *origStr,
 			 Node *escontext)
 {
-	int			i = 0,
-				nest_level = 0;
-	char	   *srcptr;
-	bool		in_quotes = false;
-	bool		eoArray = false;
-	bool		hasnull;
-	int32		totbytes;
+	int			nest_level;
+	int			maxitems;
+	char	   *elembuf;
+	bool		expect_delim;
+	Datum	   *values;
+	bool	   *nulls;
+	int			nitems;
+
+	int			ndim = *ndim_p;
+	bool		dimensions_specified = ndim != 0;
+	bool		ndim_frozen;
+	int			nelems[MAXDIM];
+
+	/* The caller already checked this */
+	Assert(**srcptr == '{');
+
+	if (!dimensions_specified)
+	{
+		/* Initialize dim[] entries to -1 meaning "unknown" */
+		for (int i = 0; i < MAXDIM; ++i)
+			dim[i] = -1;
+	}
+	ndim_frozen = dimensions_specified;
+
+	maxitems = 16;
+	values = (Datum *) palloc(maxitems * sizeof(Datum));
+	nulls = (bool *) palloc(maxitems * sizeof(bool));
 
 	/*
-	 * We have to remove " and \ characters to create a clean item value to
-	 * pass to the datatype input routine.  We overwrite each item value
-	 * in-place within arrayStr to do this.  srcptr is the current scan point,
-	 * and dstptr is where we are copying to.
+	 * Allocate scratch space to hold (string representation of) one element.
 	 *
-	 * We also want to suppress leading and trailing unquoted whitespace. We
-	 * use the leadingspace flag to suppress leading space.  Trailing space is
-	 * tracked by using dstendptr to point to the last significant output
-	 * character.
+	 * We don't know how large the elements are, so be conservative and
+	 * allocate a buffer as large as the input. The string representation of
+	 * any element in the array cannot be larger than the array string itself.
 	 *
-	 * The error checking in this routine is mostly pro-forma, since we expect
-	 * that ArrayCount() already validated the string.  So we don't bother
-	 * with errdetail messages.
+	 * XXX: For a very large array, it'd probably be better to use an
+	 * expandable StringInfo. That would save memory, and avoid the strlen()
+	 * call too.
 	 */
-	srcptr = arrayStr;
-	while (!eoArray)
+	elembuf = palloc(strlen(*srcptr) + 1);
+
+	expect_delim = false;
+	nest_level = 0;
+	nitems = 0;
+	do
 	{
-		bool		itemdone = false;
-		bool		leadingspace = true;
-		bool		hasquoting = false;
-		char	   *itemstart;	/* start of de-escaped text */
-		char	   *dstptr;		/* next output point for de-escaped text */
-		char	   *dstendptr;	/* last significant output char + 1 */
+		ArrayToken	tok;
 
-		/*
-		 * Parse next array element, collecting the de-escaped text into
-		 * itemstart..dstendptr-1.
-		 *
-		 * Notice that we do not set "itemdone" until we see a separator
-		 * (typdelim character) or the array's final right brace.  Since the
-		 * array is already verified to be nonempty and rectangular, there is
-		 * guaranteed to be another element to be processed in the first case,
-		 * while in the second case of course we'll exit the outer loop.
-		 */
-		itemstart = dstptr = dstendptr = srcptr;
+		tok = ReadArrayToken(srcptr, elembuf, typdelim, origStr, escontext);
 
-		while (!itemdone)
+		switch (tok)
 		{
-			switch (*srcptr)
-			{
-				case '\0':
-					/* Signal a premature end of the string */
-					ereturn(escontext, false,
-							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-							 errmsg("malformed array literal: \"%s\"",
-									origStr)));
-					break;
-				case '\\':
-					/* Skip backslash, copy next character as-is. */
-					srcptr++;
-					if (*srcptr == '\0')
+			case ATOK_LEVEL_START:
+				if (expect_delim)
+					ereturn(escontext, false,
+							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+							 errmsg("malformed array literal: \"%s\"", origStr),
+							 errdetail("Unexpected \"%c\" character.", '{')));
+
+				/* Initialize element counting in the new level */
+				if (nest_level >= MAXDIM)
+					ereturn(escontext, false,
+							(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+							 errmsg("number of array dimensions (%d) exceeds the maximum allowed (%d)",
+									nest_level + 1, MAXDIM)));
+
+				nelems[nest_level] = 0;
+				nest_level++;
+				if (ndim < nest_level)
+				{
+					/* Can't increase ndim once it's frozen */
+					if (ndim_frozen)
+						goto dimension_error;
+					ndim = nest_level;
+				}
+				break;
+			case ATOK_LEVEL_END:
+				if (nest_level == 0)
+					ereturn(escontext, false,
+							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+							 errmsg("malformed array literal: \"%s\"", origStr),
+							 errdetail("Unexpected \"%c\" character.",
+									   '}')));
+				nest_level--;
+				if (nest_level > 0)
+				{
+					/* Nested sub-arrays count as elements of outer level */
+					nelems[nest_level - 1]++;
+					expect_delim = true;
+				}
+				else
+					expect_delim = false;
+
+				if (dim[nest_level] < 0)
+				{
+					/* Save length of first sub-array of this level */
+					dim[nest_level] = nelems[nest_level];
+				}
+				else if (nelems[nest_level] != dim[nest_level])
+				{
+					/* Subsequent sub-arrays must have same length */
+					goto dimension_error;
+				}
+				break;
+
+			case ATOK_DELIM:
+				if (!expect_delim)
+					ereturn(escontext, false,
+							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+							 errmsg("malformed array literal: \"%s\"", origStr),
+							 errdetail("Unexpected \"%c\" character.",
+									   typdelim)));
+				expect_delim = false;
+				break;
+
+			case ATOK_ELEM:
+			case ATOK_ELEM_NULL:
+				if (expect_delim)
+					ereturn(escontext, false,
+							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+							 errmsg("malformed array literal: \"%s\"", origStr),
+							 errdetail("Unexpected array element.")));
+
+				/* enlarge the values/nulls arrays if needed */
+				if (nitems == maxitems)
+				{
+#define MaxArraySize ((size_t) (MaxAllocSize / sizeof(Datum)))
+					if (maxitems == MaxArraySize)
 						ereturn(escontext, false,
-								(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-								 errmsg("malformed array literal: \"%s\"",
-										origStr)));
-					*dstptr++ = *srcptr++;
-					/* Treat the escaped character as non-whitespace */
-					leadingspace = false;
-					dstendptr = dstptr;
-					hasquoting = true;	/* can't be a NULL marker */
-					break;
-				case '"':
-					in_quotes = !in_quotes;
-					if (in_quotes)
-						leadingspace = false;
-					else
-					{
-						/*
-						 * Advance dstendptr when we exit in_quotes; this
-						 * saves having to do it in all the other in_quotes
-						 * cases.
-						 */
-						dstendptr = dstptr;
-					}
-					hasquoting = true;	/* can't be a NULL marker */
-					srcptr++;
-					break;
-				case '{':
-					if (!in_quotes)
-					{
-						nest_level++;
-						srcptr++;
-					}
-					else
-						*dstptr++ = *srcptr++;
-					break;
-				case '}':
-					if (!in_quotes)
-					{
-						if (nest_level == 0)
-							ereturn(escontext, false,
-									(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-									 errmsg("malformed array literal: \"%s\"",
-											origStr)));
-						nest_level--;
-						if (nest_level == 0)
-							eoArray = itemdone = true;
-						srcptr++;
-					}
-					else
-						*dstptr++ = *srcptr++;
-					break;
-				default:
-					if (in_quotes)
-						*dstptr++ = *srcptr++;
-					else if (*srcptr == typdelim)
-					{
-						itemdone = true;
-						srcptr++;
-					}
-					else if (scanner_isspace(*srcptr))
-					{
-						/*
-						 * If leading space, drop it immediately.  Else, copy
-						 * but don't advance dstendptr.
-						 */
-						if (leadingspace)
-							srcptr++;
-						else
-							*dstptr++ = *srcptr++;
-					}
-					else
-					{
-						*dstptr++ = *srcptr++;
-						leadingspace = false;
-						dstendptr = dstptr;
-					}
-					break;
-			}
-		}
+								(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+								 errmsg("array size exceeds the maximum allowed (%d)",
+										(int) MaxArraySize)));
+					maxitems = Min(maxitems * 2, MaxArraySize);
+					values = (Datum *) repalloc(values, maxitems * sizeof(Datum));
+					nulls = (bool *) repalloc(nulls, maxitems * sizeof(bool));
+				}
 
-		/* Terminate de-escaped string */
-		Assert(dstptr < srcptr);
-		*dstendptr = '\0';
+				if (!InputFunctionCallSafe(inputproc, tok == ATOK_ELEM_NULL ? NULL : elembuf,
+										   typioparam, typmod,
+										   escontext,
+										   &values[nitems]))
+					return false;
+				nulls[nitems] = tok == ATOK_ELEM_NULL;
+				nitems++;
 
-		/* Safety check that we don't write past the output arrays */
-		if (i >= nitems)
-			ereturn(escontext, false,
-					(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-					 errmsg("malformed array literal: \"%s\"",
-							origStr)));
+				/*
+				 * Once we have found an element, the number of dimensions can
+				 * no longer increase, and subsequent elements must all be at
+				 * the same nesting depth.
+				 */
+				ndim_frozen = true;
+				if (nest_level != ndim)
+					goto dimension_error;
+				/* Count the new element */
+				nelems[nest_level - 1]++;
+				expect_delim = true;
+				break;
 
-		/* Convert the de-escaped string into the next output array entries */
-		if (Array_nulls && !hasquoting &&
-			pg_strcasecmp(itemstart, "NULL") == 0)
-		{
-			/* it's a NULL item */
-			if (!InputFunctionCallSafe(inputproc, NULL,
-									   typioparam, typmod,
-									   escontext,
-									   &values[i]))
-				return false;
-			nulls[i] = true;
-		}
-		else
-		{
-			if (!InputFunctionCallSafe(inputproc, itemstart,
-									   typioparam, typmod,
-									   escontext,
-									   &values[i]))
+			case ATOK_ERROR:
 				return false;
-			nulls[i] = false;
 		}
+	} while (nest_level > 0);
 
-		i++;
-	}
+	pfree(elembuf);
 
-	/* Cross-check that we filled all the output array entries */
-	if (i != nitems)
-		ereturn(escontext, false,
-				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
-				 errmsg("malformed array literal: \"%s\"",
-						origStr)));
-
-	/*
-	 * Check for nulls, compute total data space needed
-	 */
-	hasnull = false;
-	totbytes = 0;
-	for (i = 0; i < nitems; i++)
-	{
-		if (nulls[i])
-			hasnull = true;
-		else
-		{
-			/* let's just make sure data is not toasted */
-			if (typlen == -1)
-				values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i]));
-			totbytes = att_addlength_datum(totbytes, typlen, values[i]);
-			totbytes = att_align_nominal(totbytes, typalign);
-			/* check for overflow of total request */
-			if (!AllocSizeIsValid(totbytes))
-				ereturn(escontext, false,
-						(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
-						 errmsg("array size exceeds the maximum allowed (%d)",
-								(int) MaxAllocSize)));
-		}
-	}
-	*hasnulls = hasnull;
-	*nbytes = totbytes;
+	*ndim_p = ndim;
+	*values_p = values;
+	*nulls_p = nulls;
+	*nitems_p = nitems;
 	return true;
+
+dimension_error:
+	if (dimensions_specified)
+		errsave(escontext,
+				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+				 errmsg("malformed array literal: \"%s\"", origStr),
+				 errdetail("Specified array dimensions do not match array contents.")));
+	else
+		errsave(escontext,
+				(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+				 errmsg("malformed array literal: \"%s\"", origStr),
+				 errdetail("Multidimensional arrays must have sub-arrays with matching dimensions.")));
+	return false;
 }
 
 
+/*
+ * ReadArrayToken
+ *	 read one token from an array value string
+ *
+ * Starts scanning from *srcptr. One return, *srcptr is updated past the the
+ * token.
+ *
+ * If the token is ATOK_ELEM, the token string is copied to *elembuf. The
+ * caller is responsible for allocating a large enough buffer!
+ */
+static ArrayToken
+ReadArrayToken(char **srcptr, char *elembuf, char typdelim,
+			   const char *origStr, Node *escontext)
+{
+	char	   *p = *srcptr;
+	char	   *dst = elembuf;
+	char	   *dstendptr;
+	bool		has_escapes;
+
+	for (;;)
+	{
+		switch (*p)
+		{
+			case '\0':
+				errsave(escontext,
+						(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+						 errmsg("malformed array literal: \"%s\"",
+								origStr)));
+				*srcptr = p;
+				return ATOK_ERROR;
+			case '{':
+				*srcptr = p + 1;
+				return ATOK_LEVEL_START;
+			case '}':
+				*srcptr = p + 1;
+				return ATOK_LEVEL_END;
+			case '"':
+				/* quoted element */
+				p++;
+				goto quoted_element;
+			default:
+				if (*p == typdelim)
+				{
+					*srcptr = p + 1;
+					return ATOK_DELIM;
+				}
+				if (scanner_isspace(*p))
+				{
+					p++;
+					continue;
+				}
+				goto unquoted_element;
+		}
+	}
+
+quoted_element:
+	for (;;)
+	{
+		switch (*p)
+		{
+			case '\0':
+				errsave(escontext,
+						(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+						 errmsg("malformed array literal: \"%s\"",
+								origStr)));
+				*srcptr = p;
+				return ATOK_ERROR;
+			case '\\':
+				/* Skip backslash, copy next character as-is. */
+				p++;
+				if (*p == '\0')
+				{
+					errsave(escontext,
+							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+							 errmsg("malformed array literal: \"%s\"",
+									origStr)));
+					*srcptr = p;
+					return ATOK_ERROR;
+				}
+				*dst++ = *p++;
+				break;
+			case '"':
+				*dst++ = '\0';
+				*srcptr = p + 1;
+				return ATOK_ELEM;
+			default:
+				*dst++ = *p++;
+		}
+	}
+
+unquoted_element:
+	dstendptr = dst;
+	has_escapes = false;
+	for (;;)
+	{
+		switch (*p)
+		{
+			case '\0':
+			case '{':
+				errsave(escontext,
+						(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+						 errmsg("malformed array literal: \"%s\"",
+								origStr)));
+				*srcptr = p;
+				return ATOK_ERROR;
+			case '\\':
+				/* Skip backslash, copy next character as-is. */
+				p++;
+				if (*p == '\0')
+				{
+					errsave(escontext,
+							(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+							 errmsg("malformed array literal: \"%s\"",
+									origStr)));
+					*srcptr = p;
+					return ATOK_ERROR;
+				}
+				*dst++ = *p++;
+				break;
+			case '"':
+				errsave(escontext,
+						(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+						 errmsg("malformed array literal: \"%s\"",
+								origStr)));
+				*srcptr = p;
+				return ATOK_ERROR;
+
+			default:
+				if (*p == typdelim || *p == '}')
+				{
+					*dstendptr = '\0';
+					*srcptr = p;
+					if (Array_nulls && !has_escapes && pg_strcasecmp(elembuf, "NULL") == 0)
+						return ATOK_ELEM_NULL;
+					else
+						return ATOK_ELEM;
+				}
+				*dst++ = *p;
+				if (!scanner_isspace(*p))
+					dstendptr = dst;
+				p++;
+		}
+	}
+}
+
 /*
  * Copy data into an array object from a temporary array of Datums.
  *
diff --git a/src/test/regress/expected/arrays.out b/src/test/regress/expected/arrays.out
index 26d4281d..93e11068 100644
--- a/src/test/regress/expected/arrays.out
+++ b/src/test/regress/expected/arrays.out
@@ -1480,7 +1480,7 @@ select E'{{1,2},\\{2,3}}'::text[];
 ERROR:  malformed array literal: "{{1,2},\{2,3}}"
 LINE 1: select E'{{1,2},\\{2,3}}'::text[];
                ^
-DETAIL:  Unexpected "\" character.
+DETAIL:  Multidimensional arrays must have sub-arrays with matching dimensions.
 select '{{"1 2" x},{3}}'::text[];
 ERROR:  malformed array literal: "{{"1 2" x},{3}}"
 LINE 1: select '{{"1 2" x},{3}}'::text[];
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index f3d8a2a8..308ec440 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -148,8 +148,8 @@ ArrayIOData
 ArrayIterator
 ArrayMapState
 ArrayMetaState
-ArrayParseState
 ArraySubWorkspace
+ArrayToken
 ArrayType
 AsyncQueueControl
 AsyncQueueEntry
-- 
2.34.1

