Upgrade to Postgres 16

.github/workflows/ci.yml

@@ -20,20 +20,20 @@ jobs:
       id: cache-protobuf
       uses: actions/cache@v1
       with:
-        path: protobuf-3.14.0
-        key: ${{ runner.os }}-protobuf-cpp-3.14.0
+        path: protobuf-25.1
+        key: ${{ runner.os }}-protobuf-25.1
     - name: Build protobuf library
       if: matrix.protobuf_lib == 'protobuf-cpp' && steps.cache-protobuf.outputs.cache-hit != 'true'
       run: |
-        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/protobuf-cpp-3.14.0.tar.gz
-        tar -xf protobuf-cpp-3.14.0.tar.gz
-        cd protobuf-3.14.0
-        ./configure
-        make
+        git clone --depth 1 --branch v25.1 https://github.com/protocolbuffers/protobuf.git protobuf-25.1
+        cd protobuf-25.1
+        git submodule update --init --recursive
+        cmake .
+        cmake --build . --parallel 10
     - name: Install protobuf library
       if: matrix.protobuf_lib == 'protobuf-cpp'
       run: |
-        cd protobuf-3.14.0
+        cd protobuf-25.1
         sudo make install
         sudo ldconfig
     - name: Install Valgrind

Makefile

@@ -6,9 +6,9 @@ ARLIB = lib$(TARGET).a
 PGDIR = $(root_dir)/tmp/postgres
 PGDIRBZ2 = $(root_dir)/tmp/postgres.tar.bz2
 
-PG_VERSION = 15.1
+PG_VERSION = 16.1
 PG_VERSION_MAJOR = $(call word-dot,$(PG_VERSION),1)
-PROTOC_VERSION = 3.14.0
+PROTOC_VERSION = 25.1
 
 VERSION = 4.2.3
 VERSION_MAJOR = $(call word-dot,$(VERSION),1)
@@ -30,12 +30,12 @@ else
 endif
 
 SRC_FILES := $(wildcard src/*.c src/postgres/*.c) vendor/protobuf-c/protobuf-c.c vendor/xxhash/xxhash.c protobuf/pg_query.pb-c.c
-NOT_OBJ_FILES := src/pg_query_enum_defs.o src/pg_query_fingerprint_defs.o src/pg_query_fingerprint_conds.o src/pg_query_outfuncs_defs.o src/pg_query_outfuncs_conds.o src/pg_query_readfuncs_defs.o src/pg_query_readfuncs_conds.o src/postgres/guc-file.o src/postgres/scan.o src/pg_query_json_helper.o
+NOT_OBJ_FILES := src/pg_query_enum_defs.o src/pg_query_fingerprint_defs.o src/pg_query_fingerprint_conds.o src/pg_query_outfuncs_defs.o src/pg_query_outfuncs_conds.o src/pg_query_readfuncs_defs.o src/pg_query_readfuncs_conds.o src/postgres/guc-file.o src/postgres/scan.o src/pg_query_json_helper.o $(patsubst %.c,%.o,$(wildcard src/postgres/src_backend_nodes_*.funcs.c))
 OBJ_FILES := $(filter-out $(NOT_OBJ_FILES), $(SRC_FILES:.c=.o))
 
 override CFLAGS += -g -I. -I./vendor -I./src/postgres/include -Wall -Wno-unused-function -Wno-unused-value -Wno-unused-variable -fno-strict-aliasing -fwrapv -fPIC
 
-override PG_CONFIGURE_FLAGS += -q --without-readline --without-zlib
+override PG_CONFIGURE_FLAGS += -q --without-readline --without-zlib --without-icu
 
 override TEST_CFLAGS += -g -I. -I./vendor -Wall
 override TEST_LDFLAGS += -pthread
@@ -80,13 +80,13 @@ CC ?= cc
 # Experimental use of Protobuf C++ library, primarily used to validate JSON output matches Protobuf JSON mapping
 CXX_SRC_FILES := src/pg_query_outfuncs_protobuf_cpp.cc protobuf/pg_query.pb.cc
 ifeq ($(USE_PROTOBUF_CPP),1)
-	override CXXFLAGS += `pkg-config --cflags protobuf` -I. -I./src/postgres/include -DHAVE_PTHREAD -std=c++11 -Wall -Wno-unused-function -Wno-zero-length-array -Wno-c99-extensions -fwrapv -fPIC
+	override CXXFLAGS += `pkg-config --cflags protobuf` -I. -I./src/postgres/include -DHAVE_PTHREAD -std=c++17 -Wall -Wno-unused-function -Wno-zero-length-array -Wno-c99-extensions -fwrapv -fPIC
 	ifeq ($(DEBUG),1)
 		override CXXFLAGS += -O0 -g
 	else
 		override CXXFLAGS += -O3 -g
 	endif
-	override TEST_LDFLAGS += `pkg-config --libs protobuf` -lstdc++
+	override TEST_LDFLAGS += `pkg-config --libs protobuf` -lstdc++ -lm
 
 	# Don't use regular Protobuf-C or JSON implementation (instead implement the same methods using the C++ library)
 	SRC_FILES := $(filter-out src/pg_query_outfuncs_json.c src/pg_query_outfuncs_protobuf.c, $(SRC_FILES))
@@ -120,7 +120,7 @@ $(PGDIR):
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/06_alloc_set_delete_free_list.patch
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/07_plpgsql_start_finish_datums.patch
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/08_avoid_zero_length_delimiter_in_regression_tests.patch
-	cd $(PGDIR); patch -p1 < $(root_dir)/patches/09_allow_trailing_junk.patch
+	cd $(PGDIR); patch -p1 < $(root_dir)/patches/09_allow_param_junk.patch
 	cd $(PGDIR); ./configure $(PG_CONFIGURE_FLAGS)
 	cd $(PGDIR); rm src/pl/plpgsql/src/pl_gram.h
 	cd $(PGDIR); make -C src/pl/plpgsql/src pl_gram.h

patches/01_parser_additional_param_ref_support.patch

@@ -23,10 +23,10 @@ Date:   Sun Jan 3 15:06:25 2021 -0800
     SELECT INTERVAL (6) $1;
 
 diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
-index 74339aa4db..4529fea74e 100644
+index c31b373358..164b9586c1 100644
 --- a/src/backend/parser/gram.y
 +++ b/src/backend/parser/gram.y
-@@ -168,6 +168,8 @@ static Node *makeBitStringConst(char *str, int location);
+@@ -178,6 +178,8 @@ static Node *makeBoolAConst(bool state, int location);
  static Node *makeBitStringConst(char *str, int location);
  static Node *makeNullAConst(int location);
  static Node *makeAConst(Node *v, int location);
@@ -35,9 +35,9 @@ index 74339aa4db..4529fea74e 100644
  static RoleSpec *makeRoleSpec(RoleSpecType type, int location);
  static void check_qualified_name(List *names, core_yyscan_t yyscanner);
  static List *check_func_name(List *names, core_yyscan_t yyscanner);
-@@ -1086,6 +1088,11 @@ AlterOptRoleElem:
+@@ -1176,6 +1178,11 @@ AlterOptRoleElem:
  					$$ = makeDefElem("password",
- 									 (Node *)makeString($2), @1);
+ 									 (Node *) makeString($2), @1);
  				}
 +			| PASSWORD PARAM
 +				{
@@ -47,9 +47,9 @@ index 74339aa4db..4529fea74e 100644
  			| PASSWORD NULL_P
  				{
  					$$ = makeDefElem("password", NULL, @1);
-@@ -1100,6 +1107,16 @@ AlterOptRoleElem:
+@@ -1190,6 +1197,16 @@ AlterOptRoleElem:
  					$$ = makeDefElem("password",
- 									 (Node *)makeString($3), @1);
+ 									 (Node *) makeString($3), @1);
  				}
 +			| ENCRYPTED PASSWORD PARAM
 +				{
@@ -64,7 +64,7 @@ index 74339aa4db..4529fea74e 100644
  			| UNENCRYPTED PASSWORD Sconst
  				{
  					ereport(ERROR,
-@@ -1587,6 +1604,14 @@ set_rest_more:	/* Generic SET syntaxes: */
+@@ -1703,6 +1720,14 @@ set_rest_more:	/* Generic SET syntaxes: */
  					n->args = list_make1(makeStringConst($2, @2));
  					$$ = n;
  				}
@@ -79,7 +79,7 @@ index 74339aa4db..4529fea74e 100644
  			| NAMES opt_encoding
  				{
  					VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1606,6 +1631,14 @@ set_rest_more:	/* Generic SET syntaxes: */
+@@ -1724,6 +1749,14 @@ set_rest_more:	/* Generic SET syntaxes: */
  					n->args = list_make1(makeStringConst($2, @2));
  					$$ = n;
  				}
@@ -94,7 +94,7 @@ index 74339aa4db..4529fea74e 100644
  			| SESSION AUTHORIZATION NonReservedWord_or_Sconst
  				{
  					VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1614,6 +1647,14 @@ set_rest_more:	/* Generic SET syntaxes: */
+@@ -1733,6 +1766,14 @@ set_rest_more:	/* Generic SET syntaxes: */
  					n->args = list_make1(makeStringConst($3, @3));
  					$$ = n;
  				}
@@ -109,7 +109,7 @@ index 74339aa4db..4529fea74e 100644
  			| SESSION AUTHORIZATION DEFAULT
  				{
  					VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1653,6 +1694,8 @@ var_value:	opt_boolean_or_string
+@@ -1775,6 +1816,8 @@ var_value:	opt_boolean_or_string
  				{ $$ = makeStringConst($1, @1); }
  			| NumericOnly
  				{ $$ = makeAConst($1, @1); }
@@ -118,7 +118,7 @@ index 74339aa4db..4529fea74e 100644
  		;
 
  iso_level:	READ UNCOMMITTED						{ $$ = "read uncommitted"; }
-@@ -1686,6 +1729,10 @@ zone_value:
+@@ -1808,6 +1851,10 @@ zone_value:
  				{
  					$$ = makeStringConst($1, @1);
  				}
@@ -129,7 +129,7 @@ index 74339aa4db..4529fea74e 100644
  			| IDENT
  				{
  					$$ = makeStringConst($1, @1);
-@@ -14783,6 +14830,10 @@ extract_list:
+@@ -16125,6 +16172,10 @@ extract_list:
  				{
  					$$ = list_make2(makeStringConst($1, @1), $3);
  				}
@@ -138,9 +138,9 @@ index 74339aa4db..4529fea74e 100644
 +					$$ = list_make2(makeParamRef($1, @1), $3);
 +				}
  		;
-
+ 
  /* Allow delimited string Sconst in extract_arg as an SQL extension.
-@@ -15205,6 +15256,45 @@ AexprConst: Iconst
+@@ -16669,6 +16720,45 @@ AexprConst: Iconst
  					t->location = @1;
  					$$ = makeStringConstCast($6, @6, t);
  				}
@@ -186,7 +186,7 @@ index 74339aa4db..4529fea74e 100644
  			| ConstTypename Sconst
  				{
  					$$ = makeStringConstCast($2, @2, $1);
-@@ -15222,6 +15312,23 @@ AexprConst: Iconst
+@@ -16688,6 +16778,23 @@ AexprConst: Iconst
  											makeIntConst($3, @3));
  					$$ = makeStringConstCast($5, @5, t);
  				}
@@ -210,8 +210,8 @@ index 74339aa4db..4529fea74e 100644
  			| TRUE_P
  				{
  					$$ = makeBoolAConst(true, @1);
-@@ -16599,6 +1676,24 @@ makeBoolAConst(bool state, int location)
- 	return makeTypeCast((Node *)n, SystemTypeName("bool"), -1);
+@@ -18093,6 +18200,24 @@ makeAConst(Node *v, int location)
+ 	return n;
  }
 
 +/* makeParamRef

patches/03_lexer_track_yyllocend.patch

@@ -8,18 +8,18 @@ Date:   Sun Jan 3 15:59:40 2021 -0800
     as this is made available by pg_query for uses such as syntax highlighting.
 
 diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
-index a0dfcf3b99..768170bb30 100644
+index b2216a9eac..a62798cf98 100644
 --- a/src/backend/parser/scan.l
 +++ b/src/backend/parser/scan.l
-@@ -513,6 +513,7 @@ other			.
+@@ -537,6 +537,7 @@ other			.
  					{
  						/* If NCHAR isn't a keyword, just return "n" */
  						yylval->str = pstrdup("n");
 +						yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  						return IDENT;
  					}
  				}
-@@ -581,9 +582,11 @@ other			.
+@@ -605,9 +606,11 @@ other			.
  					{
  						case xb:
  							yylval->str = litbufdup(yyscanner);
@@ -31,7 +31,7 @@ index a0dfcf3b99..768170bb30 100644
  							return XCONST;
  						case xq:
  						case xe:
-@@ -596,9 +599,11 @@ other			.
+@@ -620,9 +623,11 @@ other			.
  											   yyextra->literallen,
  											   false);
  							yylval->str = litbufdup(yyscanner);
@@ -43,39 +43,39 @@ index a0dfcf3b99..768170bb30 100644
  							return USCONST;
  						default:
  							yyerror("unhandled previous state in xqs");
-@@ -736,6 +741,7 @@ other			.
+@@ -760,6 +765,7 @@ other			.
  						yyextra->dolqstart = NULL;
  						BEGIN(INITIAL);
  						yylval->str = litbufdup(yyscanner);
 +						yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  						return SCONST;
  					}
  					else
-@@ -781,6 +787,7 @@ other			.
+@@ -805,6 +811,7 @@ other			.
  					if (yyextra->literallen >= NAMEDATALEN)
  						truncate_identifier(ident, yyextra->literallen, true);
  					yylval->str = ident;
 +					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  					return IDENT;
  				}
  <xui>{dquote}	{
-@@ -789,6 +796,7 @@ other			.
+@@ -813,6 +820,7 @@ other			.
  						yyerror("zero-length delimited identifier");
  					/* can't truncate till after we de-escape the ident */
  					yylval->str = litbufdup(yyscanner);
 +					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  					return UIDENT;
  				}
  <xd,xui>{xddouble}	{
-@@ -808,6 +816,7 @@ other			.
+@@ -832,6 +840,7 @@ other			.
  					/* and treat it as {identifier} */
  					ident = downcase_truncate_identifier(yytext, yyleng, true);
  					yylval->str = ident;
 +					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  					return IDENT;
  				}
 
-@@ -1068,6 +1077,7 @@ other			.
+@@ -1096,6 +1105,7 @@ other			.
  					 */
  					ident = downcase_truncate_identifier(yytext, yyleng, true);
  					yylval->str = ident;
@@ -84,7 +84,7 @@ index a0dfcf3b99..768170bb30 100644
  				}
 
 diff --git a/src/include/parser/scanner.h b/src/include/parser/scanner.h
-index a27352afc1..29852fbd86 100644
+index da013837cd..e2a73201c4 100644
 --- a/src/include/parser/scanner.h
 +++ b/src/include/parser/scanner.h
 @@ -113,6 +113,8 @@ typedef struct core_yy_extra_type

patches/04_lexer_comments_as_tokens.patch

@@ -12,10 +12,10 @@ Date:   Sun Jan 3 16:01:44 2021 -0800
     this change, the lexer returns them as SQL_COMMENT/C_COMMENT tokens.
 
 diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
-index 8beb6fef5e..f2c963e801 100644
+index 164b9586c1..7dd5505cd1 100644
 --- a/src/backend/parser/gram.y
 +++ b/src/backend/parser/gram.y
-@@ -629,6 +629,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
+@@ -678,6 +678,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
  %token <ival>	ICONST PARAM
  %token			TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER
  %token			LESS_EQUALS GREATER_EQUALS NOT_EQUALS
@@ -24,12 +24,12 @@ index 8beb6fef5e..f2c963e801 100644
  /*
   * If you want to make any keyword changes, update the keyword table in
 diff --git a/src/backend/parser/parser.c b/src/backend/parser/parser.c
-index be86eb37fe..07b8fd486f 100644
+index e17c310cc1..9b2926878a 100644
 --- a/src/backend/parser/parser.c
 +++ b/src/backend/parser/parser.c
-@@ -150,6 +150,9 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner)
- 		case USCONST:
- 			cur_token_length = strlen(yyextra->core_yy_extra.scanbuf + *llocp);
+@@ -156,6 +156,9 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner)
+ 		case WITHOUT:
+ 			cur_token_length = 7;
  			break;
 +		case SQL_COMMENT:
 +		case C_COMMENT:
@@ -38,10 +38,10 @@ index be86eb37fe..07b8fd486f 100644
  			return cur_token;
  	}
 diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
-index 768170bb30..24fd149497 100644
+index a62798cf98..154defc6b8 100644
 --- a/src/backend/parser/scan.l
 +++ b/src/backend/parser/scan.l
-@@ -223,7 +223,7 @@ non_newline		[^\n\r]
+@@ -226,7 +226,7 @@ non_newline		[^\n\r]
 
  comment			("--"{non_newline}*)
 
@@ -50,7 +50,7 @@ index 768170bb30..24fd149497 100644
 
  /*
   * SQL requires at least one newline in the whitespace separating
-@@ -232,8 +232,8 @@ whitespace		({space}+|{comment})
+@@ -235,8 +235,8 @@ whitespace		({space}+|{comment})
   * it, whereas {whitespace} should generally have a * after it...
   */
 
@@ -61,7 +61,7 @@ index 768170bb30..24fd149497 100644
  whitespace_with_newline	({horiz_whitespace}*{newline}{special_whitespace}*)
 
  quote			'
-@@ -419,6 +419,11 @@ other			.
+@@ -443,6 +443,11 @@ other			.
  					/* ignore */
  				}
 
@@ -73,7 +73,7 @@ index 768170bb30..24fd149497 100644
  {xcstart}		{
  					/* Set location in case of syntax error in comment */
  					SET_YYLLOC();
-@@ -437,7 +442,11 @@ other			.
+@@ -461,7 +466,11 @@ other			.
 
  {xcstop}		{
  					if (yyextra->xcdepth <= 0)
@@ -86,10 +86,10 @@ index 768170bb30..24fd149497 100644
  						(yyextra->xcdepth)--;
  				}
 diff --git a/src/interfaces/ecpg/preproc/parser.c b/src/interfaces/ecpg/preproc/parser.c
-index a2eeeba217..cbfb92384a 100644
+index 38e7acb680..4814158aaf 100644
 --- a/src/interfaces/ecpg/preproc/parser.c
 +++ b/src/interfaces/ecpg/preproc/parser.c
-@@ -84,6 +84,9 @@ filtered_base_yylex(void)
+@@ -86,6 +86,9 @@ filtered_base_yylex(void)
  		case UIDENT:
  		case USCONST:
  			break;
@@ -100,10 +100,10 @@ index a2eeeba217..cbfb92384a 100644
  			return cur_token;
  	}
 diff --git a/src/pl/plpgsql/src/pl_gram.y b/src/pl/plpgsql/src/pl_gram.y
-index fc6cedb7a4..1cf9405d81 100644
+index 6a09bfdd67..fb1b519b27 100644
 --- a/src/pl/plpgsql/src/pl_gram.y
 +++ b/src/pl/plpgsql/src/pl_gram.y
-@@ -234,6 +234,7 @@ static	void			check_raise_parameters(PLpgSQL_stmt_raise *stmt);
+@@ -232,6 +232,7 @@ static	void			check_raise_parameters(PLpgSQL_stmt_raise *stmt);
  %token <ival>	ICONST PARAM
  %token			TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER
  %token			LESS_EQUALS GREATER_EQUALS NOT_EQUALS
@@ -112,7 +112,7 @@ index fc6cedb7a4..1cf9405d81 100644
  /*
   * Other tokens recognized by plpgsql's lexer interface layer (pl_scanner.c).
 diff --git a/src/pl/plpgsql/src/pl_scanner.c b/src/pl/plpgsql/src/pl_scanner.c
-index 9cea2e42ac..a0b86ac587 100644
+index 4e98af2395..40a17caaf9 100644
 --- a/src/pl/plpgsql/src/pl_scanner.c
 +++ b/src/pl/plpgsql/src/pl_scanner.c
 @@ -342,6 +342,11 @@ internal_yylex(TokenAuxData *auxdata)