fix typmod value in case of nvarchar(max), varchar(max), varbinary(ma…

…x) (#2194)

Cherry-pick commit 2d52dc0 from BABEL_3_X_DEV to BABEL_2_X_DEV

This PR fixes a crash in TdsSendTypeNVarchar for nvarchar(max) by resolving a bug in handling varchar(max) , nvarchar(max) and varbinary(max) and limits a maximum scale value for (var)char, (var)binary, and (n)(var)char types. To do that, we set a limit of 8000 for (var)char and (var)binary, and a limit of 4000 for (n)(var)char.

Task: BABEL-4547
Signed-off-by: Sandeep Kumawat <[email protected]>

contrib/babelfishpg_tds/src/backend/tds/tdsresponse.c

@@ -70,6 +70,7 @@
 #define Max(x, y)				((x) > (y) ? (x) : (y))
 #define Min(x, y)				((x) < (y) ? (x) : (y))
 #define ROWVERSION_SIZE 8
+#define VARBINARY_MAX_SCALE 8000
 
 /*
  * Local structures and functions copied from printtup.c
@@ -1860,8 +1861,13 @@ PrepareRowDescription(TupleDesc typeinfo, PlannedStmt *plannedstmt, List *target
 					if (!con->constisnull)
 					{
 						bytea	   *source = (bytea *) con->constvalue;
-
-						atttypmod = VARSIZE_ANY(source);
+						int32 actual_size = VARSIZE_ANY_EXHDR(source);
+
+						/* if the actual size is greater than 8000, it should be varbinary(max) case as we have set a limit on scale */
+						if (actual_size > VARBINARY_MAX_SCALE)
+							atttypmod = -1;
+						else 
+							atttypmod = VARSIZE_ANY(source);
 					}
 				}
 				SetColMetadataForBinaryType(col, TDS_TYPE_VARBINARY, (atttypmod == -1) ?

contrib/babelfishpg_tds/src/backend/tds/tdstypeio.c

@@ -2648,9 +2648,6 @@ TdsSendTypeVarchar(FmgrInfo *finfo, Datum value, void *vMetaData)
 	}
 	else
 	{
-		/* We can store upto 2GB (2^31 - 1 bytes) for the varchar(max). */
-		if (unlikely(actualLen > VARCHAR_MAX))
-			elog(ERROR, "Number of bytes required for the field of varchar(max) exeeds 2GB");
 		TDSInstrumentation(INSTR_TDS_DATATYPE_VARCHAR_MAX);
 
 		rc = TdsSendPlpDataHelper(destBuf, actualLen);

contrib/babelfishpg_tsql/src/pl_gram.y

@@ -54,7 +54,8 @@
  */
 #define YYMALLOC palloc
 #define YYFREE   pfree
-
+#define VARCHAR_MAX_SCALE 8000
+#define NVARCHAR_MAX_SCALE 4000
 #define TEMPOBJ_QUALIFIER "TEMPORARY "
 
 typedef struct
@@ -224,6 +225,10 @@ static	tsql_exec_param *parse_sp_proc_param(int *endtoken, bool *flag);
 static	bool word_matches_sp_proc(int tok);
 static	PLtsql_stmt *parse_sp_proc(int tok, int lineno, int return_dno);
 static	void parse_sp_cursor_value(StringInfoData* pbuffer, int *pterm);
+extern	bool is_tsql_nchar_or_nvarchar_datatype(Oid oid);
+extern	bool is_tsql_varchar_or_char_datatype(Oid oid);
+extern	bool is_tsql_binary_or_varbinary_datatype(Oid oid);
+extern	bool is_tsql_datatype_with_max_scale_expr_allowed(Oid oid);
 
 #define ereport_syntax_error(pos, msg, ...) \
 	ereport(ERROR, \
@@ -7185,9 +7190,11 @@ pltsql_sql_error_callback(void *arg)
 PLtsql_type *
 parse_datatype(const char *string, int location)
 {
-	TypeName   *typeName;
-	Oid			type_id;
+	TypeName  	*typeName;
+	Oid		type_id;
 	int32		typmod;
+	char	   *dataTypeName,
+			   *schemaName;
 	sql_error_callback_arg cbarg;
 	ErrorContextCallback  syntax_errcontext;
 
@@ -7219,6 +7226,32 @@ parse_datatype(const char *string, int location)
 	rewrite_plain_name(typeName->names);
 	typenameTypeIdAndMod(NULL, typeName, &type_id, &typmod);
 
+	/* in T-SQL, length-less (N)(VAR)CHAR's length is treated as 1 by default */
+	if (typmod == -1 && (is_tsql_varchar_or_char_datatype(type_id) || is_tsql_nchar_or_nvarchar_datatype(type_id) 
+									|| is_tsql_binary_or_varbinary_datatype(type_id)))
+		typmod = 1 + VARHDRSZ;
+
+	/* for varchar/nvarchar/varbinary(MAX), set typmod back to -1 */
+	else if (typmod == TSQLMaxTypmod && is_tsql_datatype_with_max_scale_expr_allowed(type_id))
+		typmod = -1;
+
+	else if (typmod > (VARCHAR_MAX_SCALE + VARHDRSZ) && (is_tsql_varchar_or_char_datatype(type_id) || is_tsql_binary_or_varbinary_datatype(type_id)))
+	{
+		DeconstructQualifiedName(typeName->names, &schemaName, &dataTypeName);
+		ereport(ERROR,
+			(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+			errmsg("The size '%d' exceeds the maximum allowed (%d) for '%s' datatype.",
+				typmod - VARHDRSZ, VARCHAR_MAX_SCALE, dataTypeName)));
+	}
+	else if (typmod > (NVARCHAR_MAX_SCALE + VARHDRSZ) && (is_tsql_nchar_or_nvarchar_datatype(type_id)))
+	{
+		DeconstructQualifiedName(typeName->names, &schemaName, &dataTypeName);
+		ereport(ERROR,
+			(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+			errmsg("The size '%d' exceeds the maximum allowed (%d) for '%s' datatype.",
+				typmod - VARHDRSZ, NVARCHAR_MAX_SCALE, dataTypeName)));
+	}
+
 	/* Restore former ereport callback */
 	error_context_stack = syntax_errcontext.previous;
 

contrib/babelfishpg_tsql/src/pltsql_utils.c

@@ -32,11 +32,12 @@ bool		suppress_string_truncation_error = false;
 
 bool		pltsql_suppress_string_truncation_error(void);
 
-bool		is_tsql_any_char_datatype(Oid oid); /* sys.char / sys.nchar /
-												 * sys.varchar / sys.nvarchar */
+bool		is_tsql_varchar_or_char_datatype(Oid oid); /* sys.char / sys.varchar */
+bool		is_tsql_nchar_or_nvarchar_datatype(Oid oid); /* sys.nchar / sys.nvarchar */
+bool		is_tsql_binary_or_varbinary_datatype(Oid oid); /* sys.binary / sys.varbinary */
+bool		is_tsql_datatype_with_max_scale_expr_allowed(Oid oid); /* sys.varchar(max), sys.nvarchar(max), sys.varbinary(max) */
 bool		is_tsql_text_ntext_or_image_datatype(Oid oid);
 
-bool		is_tsql_binary_or_varbinary_datatype(Oid oid);
 
 /* To cache oid of sys.varchar */
 static Oid sys_varcharoid = InvalidOid;
@@ -152,7 +153,8 @@ pltsql_check_or_set_default_typmod(TypeName *typeName, int32 *typmod, bool is_ca
 							(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
 							 errmsg("Incorrect syntax near the keyword '%s'.", typname)));
 			}
-			else if (*typmod > (max_allowed_varchar_length + VARHDRSZ) && (strcmp(typname, "varchar") == 0 || strcmp(typname, "bpchar") == 0))
+			else if (*typmod > (max_allowed_varchar_length + VARHDRSZ) && (strcmp(typname, "varchar") == 0 || strcmp(typname, "bpchar") == 0 || 
+											strcmp(typname, "varbinary") == 0 || strcmp(typname, "binary") == 0))
 			{
 				ereport(ERROR,
 						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -781,29 +783,47 @@ update_ViewStmt(Node *n, const char *view_schema)
 		stmt->view->schemaname = pstrdup(view_schema);
 }
 
+/* sys.char, sys.varchar */
 bool
-is_tsql_any_char_datatype(Oid oid)
+is_tsql_varchar_or_char_datatype(Oid oid)
 {
 	return (*common_utility_plugin_ptr->is_tsql_bpchar_datatype) (oid) ||
-		(*common_utility_plugin_ptr->is_tsql_nchar_datatype) (oid) ||
-		(*common_utility_plugin_ptr->is_tsql_varchar_datatype) (oid) ||
-		(*common_utility_plugin_ptr->is_tsql_nvarchar_datatype) (oid);
+		(*common_utility_plugin_ptr->is_tsql_varchar_datatype) (oid);
 }
 
+/* sys.nchar, sys.nvarchar */
 bool
-is_tsql_text_ntext_or_image_datatype(Oid oid)
+is_tsql_nchar_or_nvarchar_datatype(Oid oid)
 {
-	return (*common_utility_plugin_ptr->is_tsql_text_datatype) (oid) ||
-		(*common_utility_plugin_ptr->is_tsql_ntext_datatype) (oid) ||
-		(*common_utility_plugin_ptr->is_tsql_image_datatype) (oid);
+	return (*common_utility_plugin_ptr->is_tsql_nchar_datatype) (oid) ||
+		(*common_utility_plugin_ptr->is_tsql_nvarchar_datatype) (oid);
 }
 
-bool is_tsql_binary_or_varbinary_datatype(Oid oid)
+/* sys.binary, sys.varbinary */
+bool 
+is_tsql_binary_or_varbinary_datatype(Oid oid)
 {
 	return (*common_utility_plugin_ptr->is_tsql_sys_binary_datatype) (oid) ||
 		(*common_utility_plugin_ptr->is_tsql_sys_varbinary_datatype) (oid);
 }
 
+/* varchar(max), nvarchar(max), varbinary(max) */
+bool
+is_tsql_datatype_with_max_scale_expr_allowed(Oid oid)
+{
+	return	(*common_utility_plugin_ptr->is_tsql_varchar_datatype) (oid) ||
+		(*common_utility_plugin_ptr->is_tsql_nvarchar_datatype) (oid) ||
+		(*common_utility_plugin_ptr->is_tsql_sys_varbinary_datatype) (oid);
+}
+
+bool
+is_tsql_text_ntext_or_image_datatype(Oid oid)
+{
+	return (*common_utility_plugin_ptr->is_tsql_text_datatype) (oid) ||
+		(*common_utility_plugin_ptr->is_tsql_ntext_datatype) (oid) ||
+		(*common_utility_plugin_ptr->is_tsql_image_datatype) (oid);
+}
+
 /*
  * Try to acquire a lock with no wait
  */

contrib/babelfishpg_tsql/src/tsqlIface.cpp

@@ -64,9 +64,7 @@ extern "C"
 	void report_antlr_error(ANTLR_result result);
 
 	extern PLtsql_type *parse_datatype(const char *string, int location);
-	extern bool is_tsql_any_char_datatype(Oid oid);
 	extern bool is_tsql_text_ntext_or_image_datatype(Oid oid);
-	extern bool is_tsql_binary_or_varbinary_datatype(Oid oid);
 
 	extern int CurrentLineNumber;
 
@@ -4052,14 +4050,6 @@ makeDeclareStmt(TSqlParser::Declare_statementContext *ctx)
 		const char *name = downcase_truncate_identifier(nameStr.c_str(), nameStr.length(), true);
 		check_dup_declare(name);
 		PLtsql_type *type = parse_datatype(typeStr.c_str(), 0);
-
-		/* (N)(VAR)CHAR and BINARY datatype length is treated as 1 when nothing is provided */
-		if (type->atttypmod == -1 && (is_tsql_any_char_datatype(type->typoid)
-			|| is_tsql_binary_or_varbinary_datatype(type->typoid)))
-		{
-			std::string newTypeStr = typeStr + "(1)"; 
-			type = parse_datatype(newTypeStr.c_str(), 0);
-		}
 
 		PLtsql_variable *var = pltsql_build_variable(name, 0, type, true);
 
@@ -4086,15 +4076,7 @@ makeDeclareStmt(TSqlParser::Declare_statementContext *ctx)
 			{
 				std::string typeStr = ::getFullText(local->data_type());
 				PLtsql_type *type = parse_datatype(typeStr.c_str(), 0);  // FIXME: the second arg should be 'location'
-
-				/* (N)(VAR)CHAR and BINARY datatype length is treated as 1 when nothing is provided */
-				if (type->atttypmod == -1 && (is_tsql_any_char_datatype(type->typoid)
-					|| is_tsql_binary_or_varbinary_datatype(type->typoid)))
-				{
-					std::string newTypeStr = typeStr + "(1)";
-					type = parse_datatype(newTypeStr.c_str(), 0);
-				}
-				else if (is_tsql_text_ntext_or_image_datatype(type->typoid))
+				if (is_tsql_text_ntext_or_image_datatype(type->typoid))
 				{
 					throw PGErrorWrapperException(ERROR, ERRCODE_DATATYPE_MISMATCH, "The text, ntext, and image data types are invalid for local variables.", getLineAndPos(local->data_type()));
 				}