Merge branch 'main' of https://github.com/opensearch-project/dashboar…

…ds-observability into feature/multitenancy

.cypress/integration/panels_test/panels.spec.ts

@@ -388,7 +388,7 @@ describe('Panels testing with Sample Data', { defaultCommandTimeout: 10000 }, ()
         .trigger('mouseover')
         .click({ force: true })
         .focus()
-        .type(PPL_FILTER);
+        .type(PPL_FILTER, { force: true, delay: 500 });
       cy.get('button[data-test-subj="superDatePickerApplyTimeButton"]').click({ force: true });
       cy.get('.euiButton__text').contains('Refresh').trigger('mouseover').click();
       cy.get('.xtick').should('contain', 'Munich Airport');

common/constants/data_connections.ts

@@ -12,7 +12,7 @@ export const OPENSEARCH_S3_DOCUMENTATION_URL =
   'https://opensearch.org/docs/latest/dashboards/management/S3-data-source/';
 
 export const OPENSEARCH_ACC_DOCUMENTATION_URL =
-  'https://opensearch.org/docs/latest/data-acceleration/index';
+  'https://opensearch.org/docs/latest/dashboards/management/accelerate-external-data/';
 export const QUERY_RESTRICTED = 'query-restricted';
 export const QUERY_ALL = 'query-all';
 

common/constants/data_sources.ts

@@ -22,3 +22,55 @@ export enum DATA_SOURCE_TYPES {
   S3Glue = 's3glue',
 }
 export const ASYNC_POLLING_INTERVAL = 2000;
+
+export const CATALOG_CACHE_VERSION = '1.0';
+export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping';
+export const ACCELERATION_TIME_INTERVAL = [
+  { text: 'millisecond(s)', value: 'millisecond' },
+  { text: 'second(s)', value: 'second' },
+  { text: 'minutes(s)', value: 'minute' },
+  { text: 'hour(s)', value: 'hour' },
+  { text: 'day(s)', value: 'day' },
+  { text: 'week(s)', value: 'week' },
+];
+
+export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)';
+export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z][a-z_]*$/;
+export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/;
+
+export const ACCELERATION_INDEX_TYPES = [
+  { label: 'Skipping Index', value: 'skipping' },
+  { label: 'Covering Index', value: 'covering' },
+  { label: 'Materialized View', value: 'materialized' },
+];
+
+export const ACC_INDEX_TYPE_DOCUMENTATION_URL =
+  'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md';
+
+export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix_<index name>_suffix\`. They share a common prefix structure, which is \`flint_<data source name>_<database name>_<table name>_\`. Additionally, they may have a suffix that varies based on the index type. 
+##### Skipping Index
+- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`.
+  - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`.
+##### Covering Index
+- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`.
+  - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`.
+##### Materialized View Index
+- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix.
+  - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`.
+##### Note:
+- All user given index names must be in lowercase letters. Index name cannot begin with underscores. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed.  
+  `;
+
+export const SKIPPING_INDEX_ACCELERATION_METHODS = [
+  { value: 'PARTITION', text: 'Partition' },
+  { value: 'VALUE_SET', text: 'Value Set' },
+  { value: 'MIN_MAX', text: 'Min Max' },
+];
+
+export const ACCELERATION_AGGREGRATION_FUNCTIONS = [
+  { label: 'count' },
+  { label: 'sum' },
+  { label: 'avg' },
+  { label: 'max' },
+  { label: 'min' },
+];

common/constants/shared.ts

@@ -21,6 +21,7 @@ export const SAVED_OBJECTS = '/saved_objects';
 export const SAVED_QUERY = '/query';
 export const SAVED_VISUALIZATION = '/vis';
 export const CONSOLE_PROXY = '/api/console/proxy';
+export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account';
 
 // Server route
 export const PPL_ENDPOINT = '/_plugins/_ppl';
@@ -256,6 +257,8 @@ export const VISUALIZATION_ERROR = {
 export const S3_DATASOURCE_TYPE = 'S3_DATASOURCE';
 
 export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id';
+export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache';
+export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache';
 
 export const DIRECT_DUMMY_QUERY = 'select 1';
 

common/types/data_connections.ts

@@ -5,6 +5,8 @@
 
 import { EuiComboBoxOptionOption } from '@elastic/eui';
 
+export type AccelerationStatus = 'ACTIVE' | 'INACTIVE';
+
 export interface PermissionsConfigurationProps {
   roles: Role[];
   selectedRoles: Role[];
@@ -13,6 +15,216 @@ export interface PermissionsConfigurationProps {
   hasSecurityAccess: boolean;
 }
 
+export interface TableColumn {
+  name: string;
+  dataType: string;
+}
+
+export interface Acceleration {
+  name: string;
+  status: AccelerationStatus;
+  type: string;
+  database: string;
+  table: string;
+  destination: string;
+  dateCreated: number;
+  dateUpdated: number;
+  index: string;
+  sql: string;
+}
+
+export interface AssociatedObject {
+  datasource: string;
+  id: string;
+  name: string;
+  database: string;
+  type: string;
+  createdByIntegration: string;
+  accelerations: Acceleration[];
+  columns: TableColumn[];
+}
+
 export type Role = EuiComboBoxOptionOption;
 
 export type DatasourceType = 'S3GLUE' | 'PROMETHEUS';
+
+interface AsyncApiDataResponse {
+  status: string;
+  schema?: Array<{ name: string; type: string }>;
+  datarows?: any;
+  total?: number;
+  size?: number;
+  error?: string;
+}
+
+export interface AsyncApiResponse {
+  data: {
+    ok: boolean;
+    resp: AsyncApiDataResponse;
+  };
+}
+
+export type PollingCallback = (statusObj: AsyncApiResponse) => void;
+
+export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized';
+
+export type LoadCacheType = 'databases' | 'tables' | 'accelerations';
+
+export enum CachedDataSourceStatus {
+  Updated = 'Updated',
+  Failed = 'Failed',
+  Empty = 'Empty',
+}
+
+export interface CachedColumn {
+  name: string;
+  dataType: string;
+}
+
+export interface CachedTable {
+  name: string;
+  columns: CachedColumn[];
+}
+
+export interface CachedDatabase {
+  name: string;
+  tables: CachedTable[];
+  lastUpdated: string; // Assuming date string in UTC format
+  status: CachedDataSourceStatus;
+}
+
+export interface CachedDataSource {
+  name: string;
+  lastUpdated: string; // Assuming date string in UTC format
+  status: CachedDataSourceStatus;
+  databases: CachedDatabase[];
+}
+
+export interface DataSourceCacheData {
+  version: string;
+  dataSources: CachedDataSource[];
+}
+
+export interface CachedAccelerations {
+  flintIndexName: string;
+  type: AccelerationIndexType;
+  database: string;
+  table: string;
+  indexName: string;
+  autoRefresh: boolean;
+  status: string;
+}
+
+export interface AccelerationsCacheData {
+  version: string;
+  accelerations: CachedAccelerations[];
+  lastUpdated: string; // Assuming date string in UTC format
+  status: CachedDataSourceStatus;
+}
+
+export interface PollingSuccessResult {
+  schema: Array<{ name: string; type: string }>;
+  datarows: Array<Array<string | number | boolean>>;
+}
+
+export type AsyncPollingResult = PollingSuccessResult | null;
+
+export interface CreateAccelerationForm {
+  dataSource: string;
+  database: string;
+  dataTable: string;
+  dataTableFields: DataTableFieldsType[];
+  accelerationIndexType: AccelerationIndexType;
+  skippingIndexQueryData: SkippingIndexRowType[];
+  coveringIndexQueryData: string[];
+  materializedViewQueryData: MaterializedViewQueryType;
+  accelerationIndexName: string;
+  primaryShardsCount: number;
+  replicaShardsCount: number;
+  refreshType: AccelerationRefreshType;
+  checkpointLocation: string | undefined;
+  watermarkDelay: WatermarkDelayType;
+  refreshIntervalOptions: RefreshIntervalType;
+  formErrors: FormErrorsType;
+}
+
+export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min';
+
+export interface MaterializedViewColumn {
+  id: string;
+  functionName: AggregationFunctionType;
+  functionParam: string;
+  fieldAlias?: string;
+}
+
+export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX';
+
+export interface SkippingIndexRowType {
+  id: string;
+  fieldName: string;
+  dataType: string;
+  accelerationMethod: SkippingIndexAccMethodType;
+}
+
+export interface DataTableFieldsType {
+  id: string;
+  fieldName: string;
+  dataType: string;
+}
+
+export interface RefreshIntervalType {
+  refreshWindow: number;
+  refreshInterval: string;
+}
+
+export interface WatermarkDelayType {
+  delayWindow: number;
+  delayInterval: string;
+}
+
+export interface GroupByTumbleType {
+  timeField: string;
+  tumbleWindow: number;
+  tumbleInterval: string;
+}
+
+export interface MaterializedViewQueryType {
+  columnsValues: MaterializedViewColumn[];
+  groupByTumbleValue: GroupByTumbleType;
+}
+
+export interface FormErrorsType {
+  dataSourceError: string[];
+  databaseError: string[];
+  dataTableError: string[];
+  skippingIndexError: string[];
+  coveringIndexError: string[];
+  materializedViewError: string[];
+  indexNameError: string[];
+  primaryShardsError: string[];
+  replicaShardsError: string[];
+  refreshIntervalError: string[];
+  checkpointLocationError: string[];
+  watermarkDelayError: string[];
+}
+
+export type AccelerationRefreshType = 'auto' | 'interval' | 'manual';
+
+export interface CreateAccelerationForm {
+  dataSource: string;
+  database: string;
+  dataTable: string;
+  dataTableFields: DataTableFieldsType[];
+  accelerationIndexType: AccelerationIndexType;
+  skippingIndexQueryData: SkippingIndexRowType[];
+  coveringIndexQueryData: string[];
+  materializedViewQueryData: MaterializedViewQueryType;
+  accelerationIndexName: string;
+  primaryShardsCount: number;
+  replicaShardsCount: number;
+  refreshType: AccelerationRefreshType;
+  checkpointLocation: string | undefined;
+  watermarkDelay: WatermarkDelayType;
+  refreshIntervalOptions: RefreshIntervalType;
+  formErrors: FormErrorsType;
+}

common/types/explorer.ts

@@ -424,11 +424,11 @@ export interface GridSortingColumn {
 }
 
 export enum DirectQueryLoadingStatus {
-  SUCCESS = 'SUCCESS',
-  FAILED = 'FAILED',
-  RUNNING = 'RUNNING',
-  SCHEDULED = 'SCHEDULED',
-  CANCELED = 'CANCELED',
+  SUCCESS = 'success',
+  FAILED = 'failed',
+  RUNNING = 'running',
+  SCHEDULED = 'scheduled',
+  CANCELED = 'canceled',
 }
 
 export interface DirectQueryRequest {

common/utils/query_session_utils.ts

@@ -5,12 +5,12 @@
 
 import { ASYNC_QUERY_SESSION_ID } from '../constants/shared';
 
-export const setAsyncSessionId = (value: string | null) => {
+export const setAsyncSessionId = (dataSource: string, value: string | null) => {
   if (value !== null) {
-    sessionStorage.setItem(ASYNC_QUERY_SESSION_ID, value);
+    sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value);
   }
 };
 
-export const getAsyncSessionId = () => {
-  return sessionStorage.getItem(ASYNC_QUERY_SESSION_ID);
+export const getAsyncSessionId = (dataSource: string) => {
+  return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`);
 };

common/utils/shared.ts

@@ -6,3 +6,30 @@
 export function get<T = unknown>(obj: Record<string, any>, path: string, defaultValue?: T): T {
   return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue;
 }
+
+export function addBackticksIfNeeded(input: string): string {
+  // Check if the string already has backticks
+  if (input.startsWith('`') && input.endsWith('`')) {
+    return input; // Return the string as it is
+  } else {
+    // Add backticks to the string
+    return '`' + input + '`';
+  }
+}
+
+export function combineSchemaAndDatarows(
+  schema: Array<{ name: string; type: string }>,
+  datarows: Array<Array<string | number | boolean>>
+): object[] {
+  const combinedData: object[] = [];
+
+  datarows.forEach((row) => {
+    const rowData: { [key: string]: string | number | boolean } = {};
+    schema.forEach((field, index) => {
+      rowData[field.name] = row[index];
+    });
+    combinedData.push(rowData);
+  });
+
+  return combinedData;
+}