PG-Vector support in Babelfish

.github/composite-actions/build-vector-extension/action.yml

@@ -0,0 +1,22 @@
+name: 'Build vector Extension'
+
+inputs:
+  install_dir:
+    description: 'Engine install directory'
+    required: no
+    default: psql
+
+runs:
+  using: "composite"
+  steps:
+    - name: Build vector Extension
+      run: |
+        cd ..
+        export VECTOR_VERSION="0.5.1"
+        sudo apt-get install wget
+        wget https://github.com/pgvector/pgvector/archive/refs/tags/v${VECTOR_VERSION}.tar.gz
+        tar -xvzf v${VECTOR_VERSION}.tar.gz
+        cd pgvector-${VECTOR_VERSION}/
+        make USE_PGXS=1 PG_CONFIG=~/${{ inputs.install_dir }}/bin/pg_config
+        sudo make USE_PGXS=1 PG_CONFIG=~/${{ inputs.install_dir }}/bin/pg_config install
+      shell: bash

.github/composite-actions/setup-base-version/action.yml

@@ -63,6 +63,18 @@ runs:
         sudo make USE_PGXS=1 PG_CONFIG=~/${{ inputs.install_dir }}/bin/pg_config install
       shell: bash
 
+    - name: Build vector Extension
+      run: |
+        cd ..
+        export VECTOR_VERSION="0.5.1"
+        sudo apt-get install wget
+        wget https://github.com/pgvector/pgvector/archive/refs/tags/v${VECTOR_VERSION}.tar.gz 
+        tar -xvzf v${VECTOR_VERSION}.tar.gz
+        cd pgvector-${VECTOR_VERSION}/
+        make USE_PGXS=1 PG_CONFIG=~/${{ inputs.install_dir }}/bin/pg_config
+        sudo make USE_PGXS=1 PG_CONFIG=~/${{ inputs.install_dir }}/bin/pg_config install
+      shell: bash
+
     - name: Build PostGIS Extension
       run: |
         cd ..

.github/composite-actions/setup-new-version/action.yml

@@ -44,9 +44,16 @@ runs:
       with:
         install_dir: ${{ inputs.pg_new_dir }}
 
+    - name: Build vector Extension
+      id: build-vector-extension
+      if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+      uses: ./.github/composite-actions/build-vector-extension
+      with:
+        install_dir: ${{ inputs.pg_new_dir }}
+
     - name: Build PostGIS Extension
       id: build-postgis-extension
-      if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+      if: always() && steps.build-vector-extension.outcome == 'success'
       uses: ./.github/composite-actions/build-postgis-extension
       with:
         install_dir: ${{ inputs.pg_new_dir }}

.github/workflows/code-coverage.yml

@@ -44,9 +44,14 @@ jobs:
         if: always() &&  steps.build-extensions.outcome == 'success'
         uses: ./.github/composite-actions/build-tds_fdw-extension
 
+      - name: Build vector Extension
+        id: build-vector-extension
+        if: always() &&  steps.build-tds_fdw-extension.outcome == 'success'
+        uses: ./.github/composite-actions/build-vector-extension
+
       - name: Build PostGIS Extension
         id: build-postgis-extension
-        if: always() &&  steps.build-tds_fdw-extension.outcome == 'success'
+        if: always() &&  steps.build-vector-extension.outcome == 'success'
         uses: ./.github/composite-actions/build-postgis-extension
 
       - name: Install Extensions

.github/workflows/jdbc-tests-with-non-default-server-collation.yml

@@ -37,9 +37,14 @@ jobs:
         if: always() && steps.build-extensions.outcome == 'success'
         uses: ./.github/composite-actions/build-tds_fdw-extension
 
+      - name: Build vector Extension
+        id: build-vector-extension
+        if: always() &&  steps.build-tds_fdw-extension.outcome == 'success'
+        uses: ./.github/composite-actions/build-vector-extension
+
       - name: Build PostGIS Extension
         id: build-postgis-extension
-        if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+        if: always() && steps.build-vector-extension.outcome == 'success'
         uses: ./.github/composite-actions/build-postgis-extension
 
       - name: Install Extensions

.github/workflows/jdbc-tests-with-parallel-query.yml

@@ -35,9 +35,14 @@ jobs:
         if: always() && steps.build-extensions.outcome == 'success'
         uses: ./.github/composite-actions/build-tds_fdw-extension
 
+      - name: Build vector Extension
+        id: build-vector-extension
+        if: always() &&  steps.build-tds_fdw-extension.outcome == 'success'
+        uses: ./.github/composite-actions/build-vector-extension
+
       - name: Build PostGIS Extension
         id: build-postgis-extension
-        if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+        if: always() &&  steps.build-vector-extension.outcome == 'success'
         uses: ./.github/composite-actions/build-postgis-extension
 
       - name: Install Extensions

.github/workflows/jdbc-tests.yml

@@ -38,9 +38,14 @@ jobs:
         if: always() && steps.build-extensions.outcome == 'success'
         uses: ./.github/composite-actions/build-tds_fdw-extension
 
+      - name: Build vector Extension
+        id: build-vector-extension
+        if: always() &&  steps.build-tds_fdw-extension.outcome == 'success'
+        uses: ./.github/composite-actions/build-vector-extension
+
       - name: Build PostGIS Extension
         id: build-postgis-extension
-        if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+        if: always() &&  steps.build-vector-extension.outcome == 'success'
         uses: ./.github/composite-actions/build-postgis-extension
 
       - name: Install Extensions

.github/workflows/major-version-upgrade.yml

@@ -154,9 +154,16 @@ jobs:
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}
 
+      - name: Build vector Extension
+        id: build-vector-extension
+        if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+        uses: ./.github/composite-actions/build-vector-extension
+        with:
+          install_dir: ${{env.NEW_INSTALL_DIR}}
+
       - name: Build PostGIS Extension
         id: build-postgis-extension
-        if: always() && steps.build-tds_fdw-extension.outcome == 'success'
+        if: always() && steps.build-vector-extension.outcome == 'success'
         uses: ./.github/composite-actions/build-postgis-extension
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}

.github/workflows/minor-version-upgrade.yml

@@ -92,6 +92,18 @@ jobs:
           sudo make USE_PGXS=1 PG_CONFIG=~/psql/bin/pg_config install
         shell: bash
 
+      - name: Build vector Extension
+        run: |
+          cd ..
+          export VECTOR_VERSION="0.5.1"
+          sudo apt-get install wget
+          wget https://github.com/pgvector/pgvector/archive/refs/tags/v${VECTOR_VERSION}.tar.gz
+          tar -xvzf v${VECTOR_VERSION}.tar.gz
+          cd pgvector-${VECTOR_VERSION}/
+          make USE_PGXS=1 PG_CONFIG=~/psql/bin/pg_config
+          sudo make USE_PGXS=1 PG_CONFIG=~/psql/bin/pg_config install
+        shell: bash
+
       - name: Install extensions
         id: install-extensions-older
         if: always() && steps.build-tds_fdw-extension.outcome == 'success'

contrib/babelfishpg_common/src/typecode.c

@@ -51,7 +51,8 @@ type_info_t type_infos[TOTAL_TYPECODE_COUNT] =
 	{0, 1, "decimal", "decimal", 5, 30, 5},
 	{0, 1, "sysname", "sysname", 5, 31, 5},
 	{0, 1, "rowversion", "timestamp", 8, 32, 3},
-	{0, 1, "timestamp", "timestamp", 8, 33, 3}
+	{0, 1, "timestamp", "timestamp", 8, 33, 3},
+	{0, 1, "vector", "vector", 9, 34, 3}
 };
 
 /* Hash tables to help backward searching (from OID to Persist ID) */

contrib/babelfishpg_common/src/typecode.h

@@ -45,7 +45,7 @@
 #define FIXEDDECIMAL_MULTIPLIER 10000LL
 #endif
 
-#define TOTAL_TYPECODE_COUNT 33
+#define TOTAL_TYPECODE_COUNT 34
 
 struct Node;
 

contrib/babelfishpg_tds/babelfishpg_tds--1.0.0.sql

@@ -47,4 +47,4 @@ LANGUAGE SQL;
 CREATE FUNCTION sys.disable_injected_fault_all()
 RETURNS text
 AS $$ SELECT sys.inject_fault('all', 0) $$
-LANGUAGE SQL;
+LANGUAGE SQL;

contrib/babelfishpg_tds/src/backend/tds/tds_data_map.c

@@ -188,6 +188,7 @@ TdsIoFunctionRawData TdsIoFunctionRawData_data[] =
 	/* Mapping TDS listener sender to basic Postgres datatypes. */
 	{"pg_catalog", "oid", TDS_TYPE_INTEGER, 4, 1, TDS_SEND_INTEGER, TDS_RECV_INVALID},
 	{"pg_catalog", "sql_identifier", TDS_TYPE_VARCHAR, -1, 1, TDS_SEND_VARCHAR, TDS_RECV_INVALID},
+	{"sys", "vector", TDS_TYPE_VARCHAR, -1, 2, TDS_SEND_VARCHAR, TDS_RECV_VARCHAR},
 	{"pg_catalog", "name", TDS_TYPE_VARCHAR, -1, 1, TDS_SEND_VARCHAR, TDS_RECV_INVALID},
 	{"pg_catalog", "character_data", TDS_TYPE_VARCHAR, -1, 2, TDS_SEND_VARCHAR, TDS_RECV_INVALID},
 	{"pg_catalog", "bool", TDS_TYPE_BIT, 1, 1, TDS_SEND_BIT, TDS_RECV_INVALID},

contrib/babelfishpg_tsql/antlr/TSqlLexer.g4

@@ -441,6 +441,7 @@ HIGH:                                            H I G H;
 HINT:                                            H I N T;
 HISTORY_RETENTION_PERIOD:                        H I S T O R Y  UNDERLINE  R E T E N T I O N  UNDERLINE  P E R I O D;
 HISTORY_TABLE:                                   H I S T O R Y  UNDERLINE  T A B L E;
+HNSW:                                            H N S W;
 HOLDLOCK:                                        H O L D L O C K;
 HONOR_BROKER_PRIORITY:                           H O N O R  UNDERLINE  B R O K E R  UNDERLINE  P R I O R I T Y;
 HOUR:                                            H O U R;
@@ -484,6 +485,7 @@ IS:                                              I S;
 ISDESCENDANTOF:                                  I S D E S C E N D A N T O F;
 ISNULL:                                          I S N U L L;
 ISOLATION:                                       I S O L A T I O N;
+IVFFLAT:                                         I V F F L A T;
 JOB:                                             J O B;
 JOIN:                                            J O I N;
 JSON:                                            J S O N;
@@ -1055,6 +1057,9 @@ VAR:                                             V A R;
 VARBINARY_KEYWORD:                               V A R B I N A R Y;
 VARP:                                            V A R P;
 VARYING:                                         V A R Y I N G;
+VECTOR_COSINE_OPS:                               V E C T O R UNDERLINE C O S I N E UNDERLINE O P S;
+VECTOR_IP_OPS:                                   V E C T O R UNDERLINE I P UNDERLINE O P S;
+VECTOR_L2_OPS:                                   V E C T O R UNDERLINE L '2' UNDERLINE O P S;
 VERBOSELOGGING:                                  V E R B O S E L O G G I N G;
 VERSION:                                         V E R S I O N;
 VIEW:                                            V I E W;
@@ -1158,6 +1163,9 @@ MOD_ASSIGN:          '%=';
 AND_ASSIGN:          '&=';
 XOR_ASSIGN:          '^=';
 OR_ASSIGN:           '|=';
+VECTOR_L2:           '<->';
+VECTOR_COSINE:       '<=>';
+VECTOR_IP:           '<#>';
 
 DOT:                 '.';
 UNDERLINE:           '_';

contrib/babelfishpg_tsql/antlr/TSqlParser.g4

@@ -1822,9 +1822,21 @@ create_index
     (WHERE where=search_condition)?
     with_index_options?
     (ON storage_partition_clause)?
+ SEMI?
+    |CREATE UNIQUE? clustered? COLUMNSTORE? INDEX id ON table_name (USING vector_index_method)? (LR_BRACKET column_name_list_with_order_for_vector RR_BRACKET)?
+    (INCLUDE LR_BRACKET column_name_list RR_BRACKET )?
+    (WHERE where=search_condition)?
+    with_index_options?
+    (ON storage_partition_clause)?
  SEMI?
     ;
 
+/* We introduce specific index methods so as to avoid PG syntax leaks. */
+vector_index_method
+    : HNSW
+    | IVFFLAT
+    ;
+
 alter_index
     : ALTER INDEX (id | ALL) ON table_name alter_index_options
  SEMI?
@@ -3574,8 +3586,9 @@ xml_common_directives
     : COMMA (BINARY_KEYWORD BASE64 | TYPE | ROOT ( LR_BRACKET char_string RR_BRACKET )?)
     ;
 
-order_by_expression
+order_by_expression 
     : order_by=expression (ascending=ASC | descending=DESC)?
+    | order_by=expression vector_operator expression (ascending=ASC | descending=DESC)? 
     ;
 
 group_by_item
@@ -5143,6 +5156,11 @@ column_name_list_with_order
     : simple_column_name (ASC | DESC)? (COMMA simple_column_name (ASC | DESC)?)*
     ;
 
+/* We introduce specific index methods so as to avoid PG syntax leaks. */
+column_name_list_with_order_for_vector
+    : simple_column_name (ASC | DESC)? (VECTOR_COSINE_OPS | VECTOR_IP_OPS | VECTOR_L2_OPS)? (COMMA simple_column_name (ASC | DESC)? (VECTOR_COSINE_OPS | VECTOR_IP_OPS | VECTOR_L2_OPS)?)*
+    ;
+
 //For some reason, sql server allows any number of prefixes:  Here, h is the column: a.b.c.d.e.f.g.h
 insert_column_name_list
     : col+=insert_column_id (COMMA col+=insert_column_id)*
@@ -5183,7 +5201,11 @@ local_id
 // https://msdn.microsoft.com/en-us/library/ms188074.aspx
 // Spaces are allowed for comparison operators.
 comparison_operator
-    : EQUAL | GREATER | LESS | LESS EQUAL | GREATER EQUAL | LESS GREATER | EXCLAMATION EQUAL | EXCLAMATION GREATER | EXCLAMATION LESS | MULT_ASSIGN | EQUAL_STAR_OJ
+    : EQUAL | GREATER | LESS | LESS EQUAL | GREATER EQUAL | LESS GREATER | EXCLAMATION EQUAL | EXCLAMATION GREATER | EXCLAMATION LESS | MULT_ASSIGN | EQUAL_STAR_OJ | vector_operator
+    ;
+
+vector_operator
+    : VECTOR_COSINE | VECTOR_IP | VECTOR_L2
     ;
 
 assignment_operator

contrib/babelfishpg_tsql/sql/babelfishpg_tsql.sql

@@ -636,6 +636,7 @@ BEGIN
 		WHEN 'sql_variant' THEN tds_id = 98;
 		WHEN 'datetimeoffset' THEN tds_id = 43;
 		WHEN 'timestamp' THEN tds_id = 173;
+		WHEN 'vector' THEN tds_id = 167; -- Same as varchar 
 		ELSE tds_id = 0;
 	END CASE;
 	RETURN tds_id;

contrib/babelfishpg_tsql/sql/information_schema_tsql.sql

@@ -57,7 +57,7 @@ $$SELECT
 		THEN 1073741823
 		WHEN type = 'sysname'
 		THEN 128
-		WHEN type = 'xml'
+		WHEN type IN ('xml', 'vector')
 		THEN -1
 		WHEN type = 'sql_variant'
 		THEN 0

contrib/babelfishpg_tsql/sql/sys_views.sql

@@ -353,6 +353,7 @@ BEGIN
 			ELSIF typemod <= 7 THEN max_length = 5;
 			END IF;
 		WHEN 'timestamp' THEN max_length = 8;
+		WHEN 'vector' THEN max_length = -1; -- dummy as varchar max
 		ELSE max_length = typelen;
 		END CASE;
 		RETURN max_length;