Support customized alignment and storage for pg_tle create type API (#…

…266) In the previous version, we used default int4 alignment and plain storage (non-TOASTable). The data will always be stored in-line and not compressed. An error will be reported if the value exceeds the limit: ERROR: row is too big: size xxx, maximum size xxx. With this change, users can customize the alignment and storage strategies. Compression or out-of-line storage can be used depending on the storage strategy (https://www.postgresql.org/docs/current/storage-toast.html). The default alignment is int4 while the default storage is plain, compatible with previous versions.
aws · Feb 27, 2024 · 4e18a50 · 4e18a50
1 parent 93e330c
commit 4e18a50
Show file tree

Hide file tree

Showing 7 changed files with 740 additions and 13 deletions.
diff --git a/Makefile b/Makefile
@@ -1,13 +1,13 @@
 EXTENSION = pg_tle
-EXTVERSION = 1.3.4
+EXTVERSION = 1.4.0
 
 SCHEMA = pgtle
 MODULE_big = $(EXTENSION)
 
 OBJS = src/tleextension.o src/guc-file.o src/feature.o src/passcheck.o src/uni_api.o src/datatype.o src/clientauth.o
 
 EXTRA_CLEAN	= src/guc-file.c pg_tle.control pg_tle--$(EXTVERSION).sql
-DATA = pg_tle.control pg_tle--1.0.0.sql pg_tle--1.0.0--1.0.1.sql pg_tle--1.0.1--1.0.4.sql pg_tle--1.0.4.sql pg_tle--1.0.4--1.1.1.sql pg_tle--1.1.0--1.1.1.sql pg_tle--1.1.1.sql pg_tle--1.1.1--1.2.0.sql pg_tle--1.2.0--1.3.0.sql pg_tle--1.3.0--1.3.3.sql pg_tle--1.3.3--1.3.4.sql
+DATA = pg_tle.control pg_tle--1.0.0.sql pg_tle--1.0.0--1.0.1.sql pg_tle--1.0.1--1.0.4.sql pg_tle--1.0.4.sql pg_tle--1.0.4--1.1.1.sql pg_tle--1.1.0--1.1.1.sql pg_tle--1.1.1.sql pg_tle--1.1.1--1.2.0.sql pg_tle--1.2.0--1.3.0.sql pg_tle--1.3.0--1.3.3.sql pg_tle--1.3.3--1.3.4.sql pg_tle--1.3.4--1.4.0.sql
 
 TESTS = $(wildcard test/sql/*.sql)
 REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS))

diff --git a/docs/09_datatypes.md b/docs/09_datatypes.md
@@ -48,7 +48,7 @@ SELECT pgtle.create_shell_type('public', 'test_citext');
 SELECT pgtle.create_shell_type_if_not_exists('public', 'test_citext');
 ```
 
-### `pgtle.create_base_type(typenamespace regnamespace, typename name, infunc regprocedure, outfunc regprocedure, internallength int4)`
+### `pgtle.create_base_type(typenamespace regnamespace, typename name, infunc regprocedure, outfunc regprocedure, internallength int4, alignment text default 'int4', storage text default 'plain')`
 
 `create_base_type` provides a way to create a new base data type. The type must be a shell type previously defined by `create_shell_type`. This is similar to base type form of [`CREATE TYPE`](https://www.postgresql.org/docs/current/sql-createtype.html). Internally, a base data type created by `pg_tle` is stored as `bytea`, it can be cast to `bytea` explicitly after creation.
 
@@ -63,6 +63,8 @@ SELECT pgtle.create_shell_type_if_not_exists('public', 'test_citext');
 * `infunc`: The name of a previously defined function to convert the type's external textual representation to the internal representation (`bytea`). The function must take one argument of type `text` and return `bytea`. The function must also be declared as `IMMUTABLE` and `STRICT`. It will not be called with a NULL parameter.
 * `outfunc`: The name of a previously defined function to convert the type's internal binary representation (`bytea`) to the external textual representation. The function must take one argument of type `bytea` and return `text`. The function must also be declared as `IMMUTABLE` and `STRICT`. It will not be called with a NULL parameter.
 * `internallength`: Total length of the base data type as bytes. Base data types can be fixed-length, in which case internallength is a positive integer, or variable-length, in which case internallength is -1.
+* `alignment`: The optional alignment parameter specifies the storage alignment requirement of the data type. Allowed values are 'char', 'int2', 'int4', 'double'. The allowed values equate to alignment on 1, 2, 4, or 8 byte boundaries. The default is 'int4', alignment on 4 byte boundaries. A variable-length type (e.g., text) must have a byte-alignment of at least 4 due to the size of its header.
+* `storage`: The optional storage parameter allows selection of storage strategies for variable-length data types. Allowed values are 'plain', 'external', 'extended', 'main'. Only 'plain' is allowed for fixed-length types. The default is 'plain'. See PostgreSQL doc on [CREATE TYPE](https://www.postgresql.org/docs/current/sql-createtype.html) and [TOAST](https://www.postgresql.org/docs/current/storage-toast.html) for more details.
 
 #### Example
 
@@ -71,9 +73,11 @@ SELECT pgtle.create_shell_type_if_not_exists('public', 'test_citext');
 SELECT pgtle.create_base_type('public', 'test_citext', 'test_citext_in(text)'::regprocedure, 'test_citext_out(bytea)'::regprocedure, -1);
 -- Create a fixed-length (2 bytes) data type
 SELECT pgtle.create_base_type('public', 'test_int2', 'test_int2_in(text)'::regprocedure, 'test_int2_out(bytea)'::regprocedure, 2);
+-- Create a TOASTable variable-length data type 
+SELECT pgtle.create_base_type('public', 'test_citext', 'test_citext_in(text)'::regprocedure, 'test_citext_out(bytea)'::regprocedure, -1, storage => 'extended');
 ```
 
-### `pgtle.create_base_type_if_not_exists(typenamespace regnamespace, typename name, infunc regprocedure, outfunc regprocedure, internallength int4)`
+### `pgtle.create_base_type_if_not_exists(typenamespace regnamespace, typename name, infunc regprocedure, outfunc regprocedure, internallength int4, alignment text default 'int4', storage text default 'plain')`
 
 `create_base_type_if_not_exists` `create_base_type` provides a way to create a new base data type. It returns `true` if the type is created, otherwise it returns `false` if the type already exists. The type must be a shell type previously defined by `create_shell_type`. This is similar to base type form of [`CREATE TYPE`](https://www.postgresql.org/docs/current/sql-createtype.html). Internally, a base data type created by `pg_tle` is stored as `bytea`, it can be cast to `bytea` explicitly after creation.
 
@@ -88,6 +92,8 @@ SELECT pgtle.create_base_type('public', 'test_int2', 'test_int2_in(text)'::regpr
 * `infunc`: The name of a previously defined function to convert the type's external textual representation to the internal representation (`bytea`). The function must take one argument of type `text` and return `bytea`. The function must also be declared as `IMMUTABLE` and `STRICT`. It will not be called with a NULL parameter.
 * `outfunc`: The name of a previously defined function to convert the type's internal binary representation (`bytea`) to the external textual representation. The function must take one argument of type `bytea` and return `text`. The function must also be declared as `IMMUTABLE` and `STRICT`. It will not be called with a NULL parameter.
 * `internallength`: Total length of the base data type as bytes. Base data types can be fixed-length, in which case internallength is a positive integer, or variable-length, in which case internallength is -1.
+* `alignment`: The optional alignment parameter specifies the storage alignment requirement of the data type. Allowed values are 'char', 'int2', 'int4', 'double'. The allowed values equate to alignment on 1, 2, 4, or 8 byte boundaries. The default is 'int4', alignment on 4 byte boundaries. A variable-length type (e.g., text) must have a byte-alignment of at least 4 due to the size of its header.
+* `storage`: The optional storage parameter allows selection of storage strategies for variable-length data types. Allowed values are 'plain', 'external', 'extended', 'main'. Only 'plain' is allowed for fixed-length types. The default is 'plain'. See PostgreSQL doc on [CREATE TYPE](https://www.postgresql.org/docs/current/sql-createtype.html) and [TOAST](https://www.postgresql.org/docs/current/storage-toast.html) for more details.
 
 #### Example
 
@@ -96,6 +102,8 @@ SELECT pgtle.create_base_type('public', 'test_int2', 'test_int2_in(text)'::regpr
 SELECT pgtle.create_base_type_if_not_exists('public', 'test_citext', 'test_citext_in(text)'::regprocedure, 'test_citext_out(bytea)'::regprocedure, -1);
 -- Create a fixed-length (2 bytes) data type
 SELECT pgtle.create_base_type_if_not_exists('public', 'test_int2', 'test_int2_in(text)'::regprocedure, 'test_int2_out(bytea)'::regprocedure, 2);
+-- Create a TOASTable variable-length data type 
+SELECT pgtle.create_base_type_if_not_exists('public', 'test_citext', 'test_citext_in(text)'::regprocedure, 'test_citext_out(bytea)'::regprocedure, -1, storage => 'extended');
 ```
 
 ### `pgtle.create_operator_func(typenamespace regnamespace, typename name, opfunc regprocedure)`

diff --git a/include/compatibility.h b/include/compatibility.h
@@ -183,15 +183,27 @@
 #ifndef TYPALIGN_CHAR
 #define  TYPALIGN_CHAR			'c' /* char alignment (i.e. unaligned) */
 #endif
+#ifndef TYPALIGN_SHORT
+#define TYPALIGN_SHORT			's' /* short alignment (typically 2 bytes) */
+#endif
 #ifndef TYPALIGN_INT
 #define  TYPALIGN_INT			'i' /* int alignment (typically 4 bytes) */
 #endif
+#ifndef TYPALIGN_DOUBLE
+#define TYPALIGN_DOUBLE		'd' /* double alignment (often 8 bytes) */
+#endif
 #ifndef TYPSTORAGE_PLAIN
 #define  TYPSTORAGE_PLAIN		'p' /* type not prepared for toasting */
 #endif
+#ifndef TYPSTORAGE_EXTERNAL
+#define TYPSTORAGE_EXTERNAL	'e' /* toastable, don't try to compress */
+#endif
 #ifndef TYPSTORAGE_EXTENDED
 #define  TYPSTORAGE_EXTENDED	'x' /* fully toastable */
 #endif
+#ifndef TYPSTORAGE_MAIN
+#define TYPSTORAGE_MAIN		'm' /* like 'x' but try to store inline */
+#endif
 
 /*
  * PostgreSQL 13 changed the SPI interface to include a "numvals" attribute that

diff --git a/pg_tle--1.3.4--1.4.0.sql b/pg_tle--1.3.4--1.4.0.sql
@@ -0,0 +1,105 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION pg_tle" to load this file. \quit
+
+DROP FUNCTION pgtle.create_base_type CASCADE;
+DROP FUNCTION pgtle.create_base_type_if_not_exists CASCADE;
+
+CREATE FUNCTION pgtle.create_base_type
+(
+  typenamespace regnamespace,
+  typename name,
+  infunc regprocedure,
+  outfunc regprocedure,
+  internallength int4,
+  alignment text default 'int4',
+  storage text default 'plain'
+)
+RETURNS void
+SET search_path TO 'pgtle'
+STRICT
+AS 'MODULE_PATHNAME', 'pg_tle_create_base_type_with_storage'
+LANGUAGE C;
+
+CREATE FUNCTION pgtle.create_base_type_if_not_exists
+(
+  typenamespace regnamespace,
+  typename name,
+  infunc regprocedure,
+  outfunc regprocedure,
+  internallength int4,
+  alignment text default 'int4',
+  storage text default 'plain'
+)
+RETURNS boolean
+SET search_path TO 'pgtle'
+AS $_pgtleie_$
+BEGIN
+  PERFORM pgtle.create_base_type(typenamespace, typename, infunc, outfunc, internallength, alignment, storage);
+  RETURN TRUE;
+EXCEPTION
+  -- only catch the duplicate_object exception, let all other exceptions pass through.
+  WHEN duplicate_object THEN
+    RETURN FALSE;
+END;
+$_pgtleie_$
+LANGUAGE plpgsql STRICT;
+
+REVOKE EXECUTE ON FUNCTION pgtle.create_base_type
+(
+  typenamespace regnamespace,
+  typename name,
+  infunc regprocedure,
+  outfunc regprocedure,
+  internallength int4,
+  alignment text,
+  storage text
+) FROM PUBLIC;
+
+REVOKE EXECUTE ON FUNCTION pgtle.create_base_type_if_not_exists
+(
+  typenamespace regnamespace,
+  typename name,
+  infunc regprocedure,
+  outfunc regprocedure,
+  internallength int4,
+  alignment text,
+  storage text  
+) FROM PUBLIC;
+
+GRANT EXECUTE ON FUNCTION pgtle.create_base_type
+(
+  typenamespace regnamespace,
+  typename name,
+  infunc regprocedure,
+  outfunc regprocedure,
+  internallength int4,
+  alignment text,
+  storage text
+) TO pgtle_admin;
+
+GRANT EXECUTE ON FUNCTION pgtle.create_base_type_if_not_exists
+(
+  typenamespace regnamespace,
+  typename name,
+  infunc regprocedure,
+  outfunc regprocedure,
+  internallength int4,
+  alignment text,
+  storage text  
+) TO pgtle_admin;
diff --git a/src/datatype.c b/src/datatype.c
@@ -55,6 +55,17 @@ static void check_pgtle_base_type(Oid typeOid);
 static bool is_pgtle_io_func(Oid funcid, bool typeInput);
 static Oid	get_type_func_argtype(bool typeInput);
 static Oid	get_type_func_rettype(bool typeInput);
+static char get_type_alignment(char *alignmentStr);
+static char get_type_storage(char *storageStr);
+static Datum
+			pg_tle_create_base_type_internal(Oid typeNamespace,
+											 char *typeName,
+											 Oid inputFuncId,
+											 Oid outputFuncId,
+											 int16 internalLength,
+											 char *alignmentStr,
+											 char *storageStr,
+											 char *funcProbin);
 
 static void
 check_is_pgtle_admin(void)
@@ -159,6 +170,57 @@ pg_tle_create_shell_type_if_not_exists(PG_FUNCTION_ARGS)
 PG_FUNCTION_INFO_V1(pg_tle_create_base_type);
 Datum
 pg_tle_create_base_type(PG_FUNCTION_ARGS)
+{
+	Oid			typeNamespace = PG_GETARG_OID(0);
+	char	   *typeName = NameStr(*PG_GETARG_NAME(1));
+	Oid			inputFuncId = PG_GETARG_OID(2);
+	Oid			outputFuncId = PG_GETARG_OID(3);
+	int16		internalLength = PG_GETARG_INT16(4);
+	char	   *alignment = "int4"; /* default alignment */
+	char	   *storage = "plain";	/* default TOAST storage method */
+	char	   *funcProbin = get_probin(fcinfo->flinfo->fn_oid);
+
+	return pg_tle_create_base_type_internal(typeNamespace, typeName, inputFuncId, outputFuncId, internalLength, alignment, storage, funcProbin);
+}
+
+/*
+ * pg_tle_create_base_type_with_storage
+ *
+ * Similar to pg_tle_create_base_type, but expects additional arguments
+ * for alignment and storage.
+ *
+ */
+PG_FUNCTION_INFO_V1(pg_tle_create_base_type_with_storage);
+Datum
+pg_tle_create_base_type_with_storage(PG_FUNCTION_ARGS)
+{
+	Oid			typeNamespace = PG_GETARG_OID(0);
+	char	   *typeName = NameStr(*PG_GETARG_NAME(1));
+	Oid			inputFuncId = PG_GETARG_OID(2);
+	Oid			outputFuncId = PG_GETARG_OID(3);
+	int16		internalLength = PG_GETARG_INT16(4);
+	char	   *alignment = text_to_cstring(PG_GETARG_TEXT_P(5));
+	char	   *storage = text_to_cstring(PG_GETARG_TEXT_P(6));
+	char	   *funcProbin = get_probin(fcinfo->flinfo->fn_oid);
+
+	return pg_tle_create_base_type_internal(typeNamespace, typeName, inputFuncId, outputFuncId, internalLength, alignment, storage, funcProbin);
+}
+
+/*
+ * pg_tle_create_base_type_internal
+ *
+ * Internal function to create a new base type.
+ *
+ */
+static Datum
+pg_tle_create_base_type_internal(Oid typeNamespace,
+								 char *typeName,
+								 Oid inputFuncId,
+								 Oid outputFuncId,
+								 int16 internalLength,
+								 char *alignmentStr,
+								 char *storageStr,
+								 char *funcProbin)
 {
 	AclResult	aclresult;
 	Oid			inputOid;
@@ -170,12 +232,8 @@ pg_tle_create_base_type(PG_FUNCTION_ARGS)
 	Oid			inputFuncParamType;
 	Oid			outputFuncParamType;
 	char	   *namespaceName;
-	Oid			typeNamespace = PG_GETARG_OID(0);
-	char	   *typeName = NameStr(*PG_GETARG_NAME(1));
-	Oid			inputFuncId = PG_GETARG_OID(2);
-	Oid			outputFuncId = PG_GETARG_OID(3);
-	int16		internalLength = PG_GETARG_INT16(4);
-	char	   *funcProbin = get_probin(fcinfo->flinfo->fn_oid);
+	char		alignment = TYPALIGN_INT;	/* default alignment */
+	char		storage = TYPSTORAGE_PLAIN; /* default TOAST storage method */
 
 	/*
 	 * Even though the SQL function is locked down so only a member of
@@ -201,6 +259,9 @@ pg_tle_create_base_type(PG_FUNCTION_ARGS)
 	if (internalLength > 0)
 		internalLength += VARHDRSZ;
 
+	alignment = get_type_alignment(alignmentStr);
+	storage = get_type_storage(storageStr);
+
 	/* Check we have creation rights in target namespace */
 	aclresult = PG_NAMESPACE_ACLCHECK(typeNamespace, GetUserId(), ACL_CREATE);
 	namespaceName = get_namespace_name(typeNamespace);
@@ -323,8 +384,8 @@ pg_tle_create_base_type(PG_FUNCTION_ARGS)
 					NULL,		/* default type value */
 					NULL,		/* no binary form available */
 					false,		/* passed by value */
-					TYPALIGN_INT,	/* required alignment */
-					TYPSTORAGE_PLAIN,	/* TOAST strategy */
+					alignment,	/* required alignment */
+					storage,	/* TOAST strategy */
 					-1,			/* typMod (Domains only) */
 					0,			/* Array Dimensions of typbasetype */
 					false,		/* Type NOT NULL */
@@ -336,6 +397,8 @@ pg_tle_create_base_type(PG_FUNCTION_ARGS)
 	 */
 	array_type = makeArrayTypeName(typeName, typeNamespace);
 
+	/* alignment must be TYPALIGN_INT or TYPALIGN_DOUBLE for arrays */
+	alignment = (alignment == TYPALIGN_DOUBLE) ? TYPALIGN_DOUBLE : TYPALIGN_INT;
 
 	TYPE_CREATE(true,			/* array type */
 				array_oid,		/* force assignment of this type OID */
@@ -363,7 +426,7 @@ pg_tle_create_base_type(PG_FUNCTION_ARGS)
 				NULL,			/* never a default type value */
 				NULL,			/* binary default isn't sent either */
 				false,			/* never passed by value */
-				TYPALIGN_INT,	/* see above */
+				alignment,		/* see above */
 				TYPSTORAGE_EXTENDED,	/* ARRAY is always toastable */
 				-1,				/* typMod (Domains only) */
 				0,				/* Array dimensions of typbasetype */
@@ -1050,3 +1113,50 @@ pg_tle_operator_func(PG_FUNCTION_ARGS)
 
 	return OidFunctionCall2Coll(userFunc, InvalidOid, PG_GETARG_DATUM(0), PG_GETARG_DATUM(1));
 }
+
+/*
+ * get_type_alignment
+ *
+ * Get the type alignment from input string, valid options are 'double', 'int4', 'int2' and 'char'.
+ * Report an error if intput is not valid.
+ */
+static char
+get_type_alignment(char *alignmentStr)
+{
+	if (pg_strcasecmp(alignmentStr, "double") == 0)
+		return TYPALIGN_DOUBLE;
+	if (pg_strcasecmp(alignmentStr, "int4") == 0)
+		return TYPALIGN_INT;
+	if (pg_strcasecmp(alignmentStr, "int2") == 0)
+		return TYPALIGN_SHORT;
+	if (pg_strcasecmp(alignmentStr, "char") == 0)
+		return TYPALIGN_CHAR;
+
+	ereport(ERROR,
+			(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+			 errmsg("alignment \"%s\" not recognized", alignmentStr)));
+
+}
+
+/*
+ * get_type_storage
+ *
+ * Get the type storage from input string, valid options are 'plain', 'external', 'extended' and 'main'.
+ * Report an error if intput is not valid.
+ */
+static char
+get_type_storage(char *storageStr)
+{
+	if (pg_strcasecmp(storageStr, "plain") == 0)
+		return TYPSTORAGE_PLAIN;
+	if (pg_strcasecmp(storageStr, "external") == 0)
+		return TYPSTORAGE_EXTERNAL;
+	if (pg_strcasecmp(storageStr, "extended") == 0)
+		return TYPSTORAGE_EXTENDED;
+	if (pg_strcasecmp(storageStr, "main") == 0)
+		return TYPSTORAGE_MAIN;
+
+	ereport(ERROR,
+			(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+			 errmsg("storage \"%s\" not recognized", storageStr)));
+}