From 0cc144cb05b8444494ae0b8dcc8c377b61781e1c Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Thu, 7 Nov 2024 15:39:10 +0530
Subject: [PATCH 01/15] feat: re-order measures & dimensions in number & table
 chart

---
 .../charts/components/DimensionPicker.vue     | 63 ++++++++++++
 .../src2/charts/components/MeasurePicker.vue  | 99 +++++++++++++++++++
 .../components/NumberChartConfigForm.vue      | 80 ++-------------
 .../components/TableChartConfigForm.vue       | 83 +---------------
 4 files changed, 176 insertions(+), 149 deletions(-)
 create mode 100644 frontend/src2/charts/components/DimensionPicker.vue
 create mode 100644 frontend/src2/charts/components/MeasurePicker.vue

diff --git a/frontend/src2/charts/components/DimensionPicker.vue b/frontend/src2/charts/components/DimensionPicker.vue
new file mode 100644
index 00000000..8e13613b
--- /dev/null
+++ b/frontend/src2/charts/components/DimensionPicker.vue
@@ -0,0 +1,63 @@
+<script setup lang="ts">
+import { XIcon } from 'lucide-vue-next'
+import { watchEffect } from 'vue'
+import DraggableList from '../../components/DraggableList.vue'
+import { copy } from '../../helpers'
+import { Dimension } from '../../types/query.types'
+import { DimensionOption } from './ChartConfigForm.vue'
+
+const props = defineProps<{
+	options: DimensionOption[]
+}>()
+
+const dimensions = defineModel<Dimension[]>({
+	required: true,
+	default: () => [],
+})
+
+watchEffect(() => {
+	if (!dimensions.value.length) {
+		if (props.options.length) {
+			dimensions.value = [copy(props.options[0])]
+		} else {
+			addDimension()
+		}
+	}
+})
+
+function addDimension() {
+	dimensions.value.push({} as DimensionOption)
+}
+</script>
+
+<template>
+	<div>
+		<DraggableList v-model:items="dimensions">
+			<template #item="{ item, index }">
+				<div class="flex gap-1">
+					<div class="flex-1">
+						<Autocomplete
+							placeholder="Select a column"
+							:showFooter="true"
+							:options="props.options"
+							:modelValue="item.column_name"
+							@update:modelValue="Object.assign(item, $event || {})"
+						/>
+					</div>
+					<Button class="flex-shrink-0" @click="dimensions.splice(index, 1)">
+						<template #icon>
+							<XIcon class="h-4 w-4 text-gray-700" stroke-width="1.5" />
+						</template>
+					</Button>
+				</div>
+			</template>
+		</DraggableList>
+
+		<button
+			class="mt-1.5 text-left text-xs text-gray-600 hover:underline"
+			@click="addDimension"
+		>
+			+ Add column
+		</button>
+	</div>
+</template>
diff --git a/frontend/src2/charts/components/MeasurePicker.vue b/frontend/src2/charts/components/MeasurePicker.vue
new file mode 100644
index 00000000..984bbf92
--- /dev/null
+++ b/frontend/src2/charts/components/MeasurePicker.vue
@@ -0,0 +1,99 @@
+<script setup lang="ts">
+import { Settings, XIcon } from 'lucide-vue-next'
+import { watchEffect } from 'vue'
+import DraggableList from '../../components/DraggableList.vue'
+import InlineFormControlLabel from '../../components/InlineFormControlLabel.vue'
+import { copy } from '../../helpers'
+import { Measure } from '../../types/query.types'
+import { MeasureOption } from './ChartConfigForm.vue'
+
+const props = defineProps<{
+	options: MeasureOption[]
+}>()
+
+const measures = defineModel<Measure[]>({
+	required: true,
+	default: () => [],
+})
+
+watchEffect(() => {
+	if (!measures.value.length) {
+		if (props.options.length) {
+			measures.value = [copy(props.options[0])]
+		} else {
+			addMeasure()
+		}
+	}
+})
+
+function addMeasure() {
+	measures.value.push({} as MeasureOption)
+}
+</script>
+
+<template>
+	<div>
+		<DraggableList v-model:items="measures">
+			<template #item="{ item, index }">
+				<div class="flex gap-1">
+					<div class="flex-1">
+						<Autocomplete
+							placeholder="Select a column"
+							:showFooter="true"
+							:options="props.options"
+							:modelValue="item.measure_name"
+							@update:modelValue="Object.assign(item, $event || {})"
+						/>
+					</div>
+					<Popover v-if="item.measure_name" placement="bottom-end">
+						<template #target="{ togglePopover }">
+							<Button @click="togglePopover">
+								<template #icon>
+									<Settings class="h-4 w-4 text-gray-700" stroke-width="1.5" />
+								</template>
+							</Button>
+						</template>
+						<template #body-main>
+							<div class="flex w-[14rem] flex-col gap-2 p-2">
+								<InlineFormControlLabel label="Label">
+									<FormControl
+										v-model="item.measure_name"
+										autocomplete="off"
+										:debounce="500"
+									/>
+								</InlineFormControlLabel>
+
+								<slot name="measure-fields" />
+
+								<div class="flex gap-1">
+									<Button
+										class="w-full"
+										@click="measures.splice(index, 1)"
+										theme="red"
+									>
+										<template #prefix>
+											<XIcon
+												class="h-4 w-4 text-red-700"
+												stroke-width="1.5"
+											/>
+										</template>
+										Remove
+									</Button>
+								</div>
+							</div>
+						</template>
+					</Popover>
+					<Button v-else class="flex-shrink-0" @click="measures.splice(index, 1)">
+						<template #icon>
+							<XIcon class="h-4 w-4 text-gray-700" stroke-width="1.5" />
+						</template>
+					</Button>
+				</div>
+			</template>
+		</DraggableList>
+
+		<button class="mt-1.5 text-left text-xs text-gray-600 hover:underline" @click="addMeasure">
+			+ Add column
+		</button>
+	</div>
+</template>
diff --git a/frontend/src2/charts/components/NumberChartConfigForm.vue b/frontend/src2/charts/components/NumberChartConfigForm.vue
index 50d051d6..c610a1ea 100644
--- a/frontend/src2/charts/components/NumberChartConfigForm.vue
+++ b/frontend/src2/charts/components/NumberChartConfigForm.vue
@@ -1,14 +1,14 @@
 <script setup lang="ts">
-import { FIELDTYPES } from '../../helpers/constants'
-import { computed, watchEffect } from 'vue'
+import ColorInput from '@/components/Controls/ColorInput.vue'
 import { debounce } from 'frappe-ui'
+import { computed, watchEffect } from 'vue'
 import InlineFormControlLabel from '../../components/InlineFormControlLabel.vue'
+import { copy } from '../../helpers'
+import { FIELDTYPES } from '../../helpers/constants'
 import { NumberChartConfig } from '../../types/chart.types'
 import { DimensionOption, MeasureOption } from './ChartConfigForm.vue'
 import CollapsibleSection from './CollapsibleSection.vue'
-import { Settings, XIcon } from 'lucide-vue-next'
-import { copy } from '../../helpers'
-import ColorInput from '@/components/Controls/ColorInput.vue'
+import MeasurePicker from './MeasurePicker.vue'
 
 const props = defineProps<{
 	dimensions: DimensionOption[]
@@ -49,72 +49,10 @@ const updateColor = debounce((color: string) => {
 <template>
 	<CollapsibleSection title="Options">
 		<div class="flex flex-col gap-3 pt-1">
-			<template v-for="(c, idx) in config.number_columns" :key="idx">
-				<div class="flex items-end gap-1">
-					<div class="flex-1">
-						<Autocomplete
-							placeholder="Select a column"
-							:label="
-								config.number_columns.length > 1 ? `Column ${idx + 1}` : 'Column'
-							"
-							:showFooter="true"
-							:options="props.measures"
-							:modelValue="c.measure_name"
-							@update:modelValue="Object.assign(c, $event || {})"
-						/>
-					</div>
-					<Popover v-if="c.measure_name" placement="bottom-end">
-						<template #target="{ togglePopover }">
-							<Button @click="togglePopover">
-								<template #icon>
-									<Settings class="h-4 w-4 text-gray-700" stroke-width="1.5" />
-								</template>
-							</Button>
-						</template>
-						<template #body-main>
-							<div class="flex w-[14rem] flex-col gap-2 p-2">
-								<InlineFormControlLabel label="Label">
-									<FormControl
-										v-model="c.measure_name"
-										autocomplete="off"
-										:debounce="500"
-									/>
-								</InlineFormControlLabel>
-								<div class="flex gap-1">
-									<Button
-										class="w-full"
-										@click="config.number_columns.splice(idx, 1)"
-										theme="red"
-									>
-										<template #prefix>
-											<XIcon
-												class="h-4 w-4 text-red-700"
-												stroke-width="1.5"
-											/>
-										</template>
-										Remove
-									</Button>
-								</div>
-							</div>
-						</template>
-					</Popover>
-					<Button
-						v-else
-						class="flex-shrink-0"
-						@click="config.number_columns.splice(idx, 1)"
-					>
-						<template #icon>
-							<XIcon class="h-4 w-4 text-gray-700" stroke-width="1.5" />
-						</template>
-					</Button>
-				</div>
-			</template>
-			<button
-				class="-mt-1 text-left text-xs text-gray-600 hover:underline"
-				@click="addNumberColumn"
-			>
-				+ Add column
-			</button>
+			<div>
+				<p class="mb-1.5 text-xs text-gray-600">Columns</p>
+				<MeasurePicker v-model="config.number_columns" :options="props.measures" />
+			</div>
 
 			<InlineFormControlLabel label="Date">
 				<Autocomplete
diff --git a/frontend/src2/charts/components/TableChartConfigForm.vue b/frontend/src2/charts/components/TableChartConfigForm.vue
index 74d798f1..ef067946 100644
--- a/frontend/src2/charts/components/TableChartConfigForm.vue
+++ b/frontend/src2/charts/components/TableChartConfigForm.vue
@@ -4,6 +4,8 @@ import { TableChartConfig } from '../../types/chart.types'
 import { DimensionOption, MeasureOption } from './ChartConfigForm.vue'
 import CollapsibleSection from './CollapsibleSection.vue'
 import { watchEffect } from 'vue'
+import DimensionPicker from './DimensionPicker.vue'
+import MeasurePicker from './MeasurePicker.vue'
 
 const props = defineProps<{
 	dimensions: DimensionOption[]
@@ -33,89 +35,14 @@ watchEffect(() => {
 
 <template>
 	<CollapsibleSection title="Rows">
-		<div class="flex flex-col gap-2">
-			<template v-for="(row, idx) in config.rows" :key="idx">
-				<div class="flex items-end gap-1 overflow-hidden">
-					<div class="flex-1">
-						<Autocomplete
-							placeholder="Select a column"
-							:showFooter="true"
-							:options="props.dimensions"
-							:modelValue="row.column_name"
-							@update:modelValue="Object.assign(row, $event || {})"
-						/>
-					</div>
-					<Button class="flex-shrink-0" @click="config.rows.splice(idx, 1)">
-						<template #icon>
-							<XIcon class="h-4 w-4 text-gray-700" stroke-width="1.5" />
-						</template>
-					</Button>
-				</div>
-			</template>
-			<button
-				class="text-left text-xs text-gray-600 hover:underline"
-				@click="config.rows.push({} as any)"
-			>
-				+ Add Row
-			</button>
-		</div>
+		<DimensionPicker v-model="config.rows" :options="props.dimensions" />
 	</CollapsibleSection>
 	<CollapsibleSection title="Columns">
-		<div class="flex flex-col gap-2">
-			<template v-for="(col, idx) in config.columns" :key="idx">
-				<div class="flex items-end gap-1 overflow-hidden">
-					<div class="flex-1">
-						<Autocomplete
-							placeholder="Select a column"
-							:showFooter="true"
-							:options="props.dimensions"
-							:modelValue="col.column_name"
-							@update:modelValue="Object.assign(col, $event || {})"
-						/>
-					</div>
-					<Button class="flex-shrink-0" @click="config.columns.splice(idx, 1)">
-						<template #icon>
-							<XIcon class="h-4 w-4 text-gray-700" stroke-width="1.5" />
-						</template>
-					</Button>
-				</div>
-			</template>
-			<button
-				class="text-left text-xs text-gray-600 hover:underline"
-				@click="config.columns.push({} as any)"
-			>
-				+ Add Column
-			</button>
-		</div>
+		<DimensionPicker v-model="config.columns" :options="props.dimensions" />
 	</CollapsibleSection>
 	<CollapsibleSection title="Values">
 		<div class="flex flex-col gap-3">
-			<div class="flex flex-col gap-2">
-				<template v-for="(val, idx) in config.values" :key="idx">
-					<div class="flex items-end gap-1 overflow-hidden">
-						<div class="flex-1">
-							<Autocomplete
-								placeholder="Select a column"
-								:showFooter="true"
-								:options="props.measures"
-								:modelValue="val.measure_name"
-								@update:modelValue="Object.assign(val, $event || {})"
-							/>
-						</div>
-						<Button class="flex-shrink-0" @click="config.values.splice(idx, 1)">
-							<template #icon>
-								<XIcon class="h-4 w-4 text-gray-700" stroke-width="1.5" />
-							</template>
-						</Button>
-					</div>
-				</template>
-				<button
-					class="text-left text-xs text-gray-600 hover:underline"
-					@click="config.values.push({} as any)"
-				>
-					+ Add Value
-				</button>
-			</div>
+			<MeasurePicker v-model="config.values" :options="props.measures" />
 			<Checkbox label="Show Filters" v-model="config.show_filter_row" />
 			<Checkbox label="Show Row Totals" v-model="config.show_row_totals" />
 			<Checkbox label="Show Column Totals" v-model="config.show_column_totals" />

From 440b9393d5fcc946e0945ebb71a44c40e832729a Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Thu, 7 Nov 2024 15:54:24 +0530
Subject: [PATCH 02/15] fix: encode passwords in connection strings

---
 .../doctype/insights_data_source_v3/connectors/mariadb.py  | 2 ++
 .../insights_data_source_v3/connectors/postgresql.py       | 7 ++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/insights/insights/doctype/insights_data_source_v3/connectors/mariadb.py b/insights/insights/doctype/insights_data_source_v3/connectors/mariadb.py
index cec2c60c..86545edd 100644
--- a/insights/insights/doctype/insights_data_source_v3/connectors/mariadb.py
+++ b/insights/insights/doctype/insights_data_source_v3/connectors/mariadb.py
@@ -1,9 +1,11 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors
 # For license information, please see license.txt
+from urllib.parse import quote_plus
 
 
 def get_mariadb_connection_string(data_source):
     password = data_source.get_password(raise_exception=False)
+    password = quote_plus(password) if password else ""
     connection_string = (
         f"mysql://{data_source.username}:{password}"
         f"@{data_source.host}:{data_source.port}/{data_source.database_name}"
diff --git a/insights/insights/doctype/insights_data_source_v3/connectors/postgresql.py b/insights/insights/doctype/insights_data_source_v3/connectors/postgresql.py
index 19f072b6..ec9912c5 100644
--- a/insights/insights/doctype/insights_data_source_v3/connectors/postgresql.py
+++ b/insights/insights/doctype/insights_data_source_v3/connectors/postgresql.py
@@ -2,11 +2,16 @@
 # For license information, please see license.txt
 
 
+from urllib.parse import quote_plus
+
+
 def get_postgres_connection_string(data_source):
     if data_source.connection_string:
-        return data_source.connection_string
+        conn_string = quote_plus(data_source.connection_string)
+        return conn_string
     else:
         password = data_source.get_password(raise_exception=False)
+        password = quote_plus(password) if password else ""
         connection_string = (
             f"postgresql://{data_source.username}:{password}"
             f"@{data_source.host}:{data_source.port}/{data_source.database_name}"

From 92940984a89a80b94ed12582e383bfc663252813 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Thu, 7 Nov 2024 17:57:03 +0530
Subject: [PATCH 03/15] refactor: simpler query table operations resolution

---
 frontend/src2/query/query.ts                  | 52 ++++---------------
 .../insights_data_source_v3/ibis_utils.py     | 43 +++++++--------
 .../insights_workbook/insights_workbook.py    | 24 +++------
 3 files changed, 37 insertions(+), 82 deletions(-)

diff --git a/frontend/src2/query/query.ts b/frontend/src2/query/query.ts
index 712c15c3..46be7d01 100644
--- a/frontend/src2/query/query.ts
+++ b/frontend/src2/query/query.ts
@@ -24,7 +24,7 @@ import {
 	Source,
 	SourceArgs,
 	SummarizeArgs,
-	UnionArgs,
+	UnionArgs
 } from '../types/query.types'
 import { WorkbookQuery } from '../types/workbook.types'
 import {
@@ -197,41 +197,9 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 			return []
 		}
 
-		const sourceOp = query.doc.operations.find((op) => op.type === 'source')
-		if (!sourceOp) {
-			return []
-		}
-
-		if ('query' in sourceOp && sourceOp.query) {
-			// move old structure to new structure
-			sourceOp.table = query_table({
-				query_name: sourceOp.query as string,
-			})
-			delete sourceOp.query
-		}
-
 		let _operations = [...query.currentOperations]
-
-		if (sourceOp.table.type === 'query') {
-			const sourceQuery = getCachedQuery(sourceOp.table.query_name)
-			if (!sourceQuery) {
-				const message = `Source query ${sourceOp.table.query_name} not found`
-				createToast({
-					variant: 'error',
-					title: 'Error',
-					message,
-				})
-				throw new Error(message)
-			}
-
-			const sourceQueryOperations = sourceQuery.getOperationsForExecution()
-			const currentOperationsWithoutSource = query.currentOperations.slice(1)
-
-			_operations = [...sourceQueryOperations, ...currentOperationsWithoutSource]
-		}
-
 		for (const op of _operations) {
-			if (op.type !== 'join' && op.type !== 'union') continue
+			if (op.type !== 'source' && op.type !== 'join' && op.type !== 'union') continue
 			if (op.table.type !== 'query') continue
 
 			const queryTable = getCachedQuery(op.table.query_name)
@@ -270,6 +238,7 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 				query.result.executedSQL = response.sql
 				query.result.columns = response.columns
 				query.result.rows = response.rows
+				query.result.totalRowCount = 0
 				query.result.formattedRows = getFormattedRows(query.result, query.doc.operations)
 				query.result.columnOptions = query.result.columns.map((column) => ({
 					label: column.name,
@@ -297,13 +266,13 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 			use_live_connection: query.doc.use_live_connection,
 			operations,
 		})
-		.then((count: number) => {
-			query.result.totalRowCount = count || 0
-		})
-		.catch(showErrorToast)
-		.finally(() => {
-			query.fetchingCount = false
-		})
+			.then((count: number) => {
+				query.result.totalRowCount = count || 0
+			})
+			.catch(showErrorToast)
+			.finally(() => {
+				query.fetchingCount = false
+			})
 	}
 
 	function setActiveOperation(index: number) {
@@ -664,7 +633,6 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 			query.activeEditOperation.type === 'summarize'
 		) {
 			// remove the active edit operation from the operations
-			// we can't use activeEditIndex because the operations for execution may have more operations (after query table resolution)
 			const idx = operationsForExecution.findIndex((op) => op === query.activeEditOperation)
 			operationsForExecution.splice(idx, 1)
 		}
diff --git a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
index d20c8061..ee05ffc5 100644
--- a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
+++ b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
@@ -32,12 +32,25 @@ def build(self, operations: list, use_live_connection=True) -> IbisQuery:
             self.query = self.perform_operation(operation)
         return self.query
 
-    def get_table(self, table):
-        return InsightsTablev3.get_ibis_table(
-            table.data_source,
-            table.table_name,
-            use_live_connection=self.use_live_connection,
-        )
+    def get_table_or_query(self, table_args):
+        _table = None
+
+        if table_args.type == "table":
+            _table = InsightsTablev3.get_ibis_table(
+                table_args.data_source,
+                table_args.table_name,
+                use_live_connection=self.use_live_connection,
+            )
+        if table_args.type == "query":
+            _table = IbisQueryBuilder().build(
+                table_args.operations,
+                use_live_connection=self.use_live_connection,
+            )
+
+        if _table is None:
+            frappe.throw("Invalid join table")
+
+        return _table
 
     def perform_operation(self, operation):
         operation = _dict(operation)
@@ -74,7 +87,7 @@ def perform_operation(self, operation):
         return self.query
 
     def apply_source(self, source_args):
-        return self.get_table(source_args.table)
+        return self.get_table_or_query(source_args.table)
 
     def apply_join(self, join_args):
         right_table = self.get_right_table(join_args)
@@ -87,22 +100,6 @@ def apply_join(self, join_args):
             how=join_type,
         )
 
-    def get_table_or_query(self, table_args):
-        _table = None
-
-        if table_args.type == "table":
-            _table = self.get_table(table_args)
-        if table_args.type == "query":
-            _table = IbisQueryBuilder().build(
-                table_args.operations,
-                use_live_connection=self.use_live_connection,
-            )
-
-        if _table is None:
-            frappe.throw("Invalid join table")
-
-        return _table
-
     def get_right_table(self, join_args):
         right_table = self.get_table_or_query(join_args.table)
 
diff --git a/insights/insights/doctype/insights_workbook/insights_workbook.py b/insights/insights/doctype/insights_workbook/insights_workbook.py
index c0b5210c..184c0fb6 100644
--- a/insights/insights/doctype/insights_workbook/insights_workbook.py
+++ b/insights/insights/doctype/insights_workbook/insights_workbook.py
@@ -122,25 +122,15 @@ def can_access_shared_chart(self, chart):
         return False
 
     def resolve_query_tables(self, operations):
-        source_op = next((op for op in operations if op["type"] == "source"), None)
-        if not source_op:
-            return []
-
-        if source_op["table"]["type"] == "query":
-            query_name = source_op["table"]["query_name"]
-            source_query = self.query_map.get(query_name)
-            if not source_query:
-                frappe.throw(f"Source query {query_name} not found")
-
-            source_query_operations = self.resolve_query_tables(
-                source_query["operations"]
-            )
-            current_operations_without_source = operations[1:]
-
-            operations = source_query_operations + current_operations_without_source
+        if not operations:
+            return operations
 
         for op in operations:
-            if op["type"] != "join" and op["type"] != "union":
+            if (
+                op["type"] != "source"
+                and op["type"] != "join"
+                and op["type"] != "union"
+            ):
                 continue
             if op["table"]["type"] != "query":
                 continue

From 3f03b2337e9055671a6801f55c07e4ab87744b15 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Thu, 7 Nov 2024 17:57:19 +0530
Subject: [PATCH 04/15] chore: prevent invalid table imports

---
 .../doctype/insights_data_source_v3/data_warehouse.py     | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/insights/insights/doctype/insights_data_source_v3/data_warehouse.py b/insights/insights/doctype/insights_data_source_v3/data_warehouse.py
index 58e90e73..8d9985f5 100644
--- a/insights/insights/doctype/insights_data_source_v3/data_warehouse.py
+++ b/insights/insights/doctype/insights_data_source_v3/data_warehouse.py
@@ -46,6 +46,14 @@ def __init__(self, data_source: str, table_name: str):
         self.warehouse_table_name = get_warehouse_table_name(data_source, table_name)
         self.parquet_filepath = get_parquet_filepath(data_source, table_name)
 
+        self.validate()
+
+    def validate(self):
+        if not self.data_source:
+            frappe.throw("Data Source is required.")
+        if not self.table_name:
+            frappe.throw("Table Name is required.")
+
     def get_ibis_table(self, import_if_not_exists: bool = True) -> Expr:
         if not os.path.exists(self.parquet_filepath):
             if import_if_not_exists:

From f10772b1a67745c7dd367c013a951e41be036ee8 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Thu, 7 Nov 2024 18:30:41 +0530
Subject: [PATCH 05/15] feat: apply dashboard filters to queries

---
 .../dashboard/DashboardFilterSelector.vue     | 12 ++++---
 frontend/src2/dashboard/dashboard.ts          | 34 ++++++++-----------
 frontend/src2/query/components/FilterRule.vue |  8 +++--
 .../src2/query/components/FiltersSelector.vue | 31 +++++++++--------
 .../components/FiltersSelectorDialog.vue      |  4 +++
 frontend/src2/query/query.ts                  |  8 +++++
 6 files changed, 56 insertions(+), 41 deletions(-)

diff --git a/frontend/src2/dashboard/DashboardFilterSelector.vue b/frontend/src2/dashboard/DashboardFilterSelector.vue
index 103b30f4..a58a05df 100644
--- a/frontend/src2/dashboard/DashboardFilterSelector.vue
+++ b/frontend/src2/dashboard/DashboardFilterSelector.vue
@@ -42,6 +42,7 @@ if (chartQueries.value.length) {
 	})
 }
 
+const sep = '`'
 const columnOptions = computed(() => {
 	return chartQueries.value
 		.map((q) => {
@@ -52,7 +53,7 @@ const columnOptions = computed(() => {
 				label: c.name,
 				data_type: c.type,
 				description: c.type,
-				value: `'${q.name}'.'${c.name}'`,
+				value: `${sep}${q.name}${sep}.${sep}${c.name}${sep}`,
 			}))
 			return {
 				group: query.doc.title || q.name,
@@ -74,7 +75,8 @@ function setInitialFilters() {
 		.map(([queryName, filters]) => {
 			return filters.map((filter) => {
 				if ('column' in filter) {
-					filter.column.column_name = `'${queryName}'.'${filter.column.column_name}'`
+					const col_name = filter.column.column_name
+					filter.column.column_name = `${sep}${queryName}${sep}.${sep}${col_name}${sep}`
 				}
 				return filter
 			})
@@ -87,8 +89,8 @@ function applyFilters(args: FilterGroupArgs) {
 	args.filters.forEach((filter) => {
 		if ('column' in filter) {
 			const [queryName, columnName] = filter.column.column_name
-				.split("'.'")
-				.map((s) => s.replace(/'/g, ''))
+				.split(`${sep}.${sep}`)
+				.map((s) => s.replace(new RegExp(sep, 'g'), ''))
 
 			filter.column.column_name = columnName
 			filtersByQuery[queryName] = filtersByQuery[queryName] || []
@@ -121,6 +123,8 @@ function applyFilters(args: FilterGroupArgs) {
 		v-model="showDialog"
 		:filter-group="filterGroup"
 		:column-options="columnOptions"
+		:disable-logical-operator="true"
+		:disable-expressions="true"
 		@select="applyFilters($event)"
 	/>
 </template>
diff --git a/frontend/src2/dashboard/dashboard.ts b/frontend/src2/dashboard/dashboard.ts
index 080d5fb1..10bbad4c 100644
--- a/frontend/src2/dashboard/dashboard.ts
+++ b/frontend/src2/dashboard/dashboard.ts
@@ -1,12 +1,11 @@
 import { reactive } from 'vue'
 import { getCachedChart } from '../charts/chart'
 import { getUniqueId, store } from '../helpers'
+import { getCachedQuery } from '../query/query'
 import { FilterArgs } from '../types/query.types'
 import {
-	DashboardFilterColumn,
 	WorkbookChart,
-	WorkbookDashboard,
-	WorkbookDashboardChart,
+	WorkbookDashboard
 } from '../types/workbook.types'
 
 const dashboards = new Map<string, Dashboard>()
@@ -61,20 +60,6 @@ function makeDashboard(workbookDashboard: WorkbookDashboard) {
 			})
 		},
 
-		addFilter(column: DashboardFilterColumn) {
-			dashboard.doc.items.push({
-				type: 'filter',
-				column: column,
-				layout: {
-					i: getUniqueId(),
-					x: 0,
-					y: 0,
-					w: 4,
-					h: 1,
-				},
-			})
-		},
-
 		removeItem(index: number) {
 			dashboard.doc.items.splice(index, 1)
 		},
@@ -86,12 +71,21 @@ function makeDashboard(workbookDashboard: WorkbookDashboard) {
 
 		refresh() {
 			dashboard.doc.items
-				.filter((item): item is WorkbookDashboardChart => item.type === 'chart')
+				.filter((item) => item.type === 'chart')
 				.forEach((chartItem) => {
 					const chart = getCachedChart(chartItem.chart)
 					if (!chart || !chart.doc.query) return
-					const filters = dashboard.filters[chart.doc.query]
-					chart.refresh(filters, true)
+
+					Object.keys(dashboard.filters).forEach((query) => {
+						const _query = getCachedQuery(query)
+						if (!_query) return
+						_query.dashboardFilters = {
+							logical_operator: 'And',
+							filters: dashboard.filters[query],
+						}
+					})
+
+					chart.refresh()
 				})
 		},
 
diff --git a/frontend/src2/query/components/FilterRule.vue b/frontend/src2/query/components/FilterRule.vue
index eafd5893..027a5b57 100644
--- a/frontend/src2/query/components/FilterRule.vue
+++ b/frontend/src2/query/components/FilterRule.vue
@@ -99,8 +99,10 @@ const fetchColumnValues = debounce((searchTxt: string) => {
 		console.warn('Query not found for column:', filter.value.column.column_name)
 		return
 	}
-	// if column_name is 'query'.'column_name' extract column_name
-	const pattern = /'([^']+)'\.'([^']+)'/g
+	// only for dashboard filters
+	// if column_name is {sep}query{sep}.{sep}column_name{sep} extract column_name
+	const sep = '`'
+	const pattern = new RegExp(`${sep}([^${sep}]+)${sep}\\.${sep}([^${sep}]+)${sep}`)
 	const match = pattern.exec(filter.value.column.column_name)
 	const column_name = match ? match[2] : filter.value.column.column_name
 
@@ -172,7 +174,7 @@ const fetchColumnValues = debounce((searchTxt: string) => {
 				:options="distinctColumnValues"
 				:loading="fetchingValues"
 				@update:query="fetchColumnValues"
-				@update:modelValue="filter.value = $event.map((v: any) => v.value)"
+				@update:modelValue="filter.value = $event?.map((v: any) => v.value) || []"
 			/>
 			<FormControl v-else disabled />
 		</div>
diff --git a/frontend/src2/query/components/FiltersSelector.vue b/frontend/src2/query/components/FiltersSelector.vue
index 9076469b..21916fc5 100644
--- a/frontend/src2/query/components/FiltersSelector.vue
+++ b/frontend/src2/query/components/FiltersSelector.vue
@@ -2,7 +2,7 @@
 import { PlusIcon, RefreshCcw } from 'lucide-vue-next'
 import { computed, reactive } from 'vue'
 import { copy, flattenOptions } from '../../helpers'
-import { Column, ColumnOption, FilterGroupArgs, GroupedColumnOption } from '../../types/query.types'
+import { ColumnOption, FilterGroupArgs, GroupedColumnOption } from '../../types/query.types'
 import { column, expression } from '../helpers'
 import FilterRule from './FilterRule.vue'
 import InlineExpression from './InlineExpression.vue'
@@ -11,6 +11,8 @@ import { isFilterExpressionValid, isFilterValid } from './filter_utils'
 const props = defineProps<{
 	filterGroup?: FilterGroupArgs
 	columnOptions: ColumnOption[] | GroupedColumnOption[]
+	disableLogicalOperator?: boolean
+	disableExpressions?: boolean
 }>()
 
 const emit = defineEmits({
@@ -75,8 +77,11 @@ const areFiltersUpdated = computed(() => {
 						v-else
 						class="text-sm"
 						@click="
-							filterGroup.logical_operator =
-								filterGroup.logical_operator === 'And' ? 'Or' : 'And'
+							() => {
+								if (props.disableLogicalOperator) return
+								filterGroup.logical_operator =
+									filterGroup.logical_operator === 'And' ? 'Or' : 'And'
+							}
 						"
 					>
 						<template #prefix>
@@ -104,20 +109,18 @@ const areFiltersUpdated = computed(() => {
 						variant: 'ghost',
 					}"
 					:options="[
-						{
-							label: 'Convert to Expression',
-							onClick: () => {
-								filterGroup.filters[i] = { expression: expression('') }
-							},
-						},
+						!props.disableExpressions
+							? {
+									label: 'Convert to Expression',
+									onClick: () => {
+										filterGroup.filters[i] = { expression: expression('') }
+									},
+							  }
+							: null,
 						{
 							label: 'Duplicate',
 							onClick: () => {
-								filterGroup.filters.splice(
-									i + 1,
-									0,
-									JSON.parse(JSON.stringify(filterGroup.filters[i]))
-								)
+								filterGroup.filters.splice(i + 1, 0, copy(filterGroup.filters[i]))
 							},
 						},
 						{
diff --git a/frontend/src2/query/components/FiltersSelectorDialog.vue b/frontend/src2/query/components/FiltersSelectorDialog.vue
index 040ef7b3..d4a43558 100644
--- a/frontend/src2/query/components/FiltersSelectorDialog.vue
+++ b/frontend/src2/query/components/FiltersSelectorDialog.vue
@@ -5,6 +5,8 @@ import FiltersSelector from './FiltersSelector.vue'
 const props = defineProps<{
 	filterGroup?: FilterGroupArgs
 	columnOptions: ColumnOption[] | GroupedColumnOption[]
+	disableLogicalOperator?: boolean
+	disableExpressions?: boolean
 }>()
 const emit = defineEmits({ select: (args: FilterGroupArgs) => true })
 const showDialog = defineModel()
@@ -16,6 +18,8 @@ const showDialog = defineModel()
 			<FiltersSelector
 				:filterGroup="props.filterGroup"
 				:columnOptions="props.columnOptions"
+				:disableLogicalOperator="props.disableLogicalOperator"
+				:disableExpressions="props.disableExpressions"
 				@close="showDialog = false"
 				@select="
 					(args) => {
diff --git a/frontend/src2/query/query.ts b/frontend/src2/query/query.ts
index 46be7d01..4e47a703 100644
--- a/frontend/src2/query/query.ts
+++ b/frontend/src2/query/query.ts
@@ -118,6 +118,8 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 		reorderOperations,
 		reset,
 
+		dashboardFilters: {} as FilterGroupArgs,
+
 		history: {} as UseRefHistoryReturn<any, any>,
 		canUndo() {
 			return !query.activeEditIndex && !query.executing
@@ -216,6 +218,12 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 			op.table.operations = queryTable.getOperationsForExecution()
 		}
 
+
+		if (query.dashboardFilters.filters?.length) {
+			_operations.push(filter_group(query.dashboardFilters))
+			query.dashboardFilters = {} as FilterGroupArgs
+		}
+
 		return _operations
 	}
 

From f947f474c099f8d5a005063845da9943a6fcbb0c Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Thu, 7 Nov 2024 18:43:26 +0530
Subject: [PATCH 06/15] revert: `replace NaNs before storing results`

---
 insights/insights/doctype/insights_query/insights_query.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/insights/insights/doctype/insights_query/insights_query.py b/insights/insights/doctype/insights_query/insights_query.py
index f5aa4a0c..84e57f92 100644
--- a/insights/insights/doctype/insights_query/insights_query.py
+++ b/insights/insights/doctype/insights_query/insights_query.py
@@ -7,8 +7,6 @@
 from functools import cached_property
 
 import frappe
-import numpy as np
-import pandas as pd
 from frappe import _dict
 from frappe.model.document import Document
 from frappe.utils import flt
@@ -220,8 +218,6 @@ def fetch_results(self, additional_filters=None):
     def update_query_results(self, results=None):
         results = results or []
         query_result: Document = InsightsQueryResult.get_or_create_doc(query=self.name)
-        results = pd.DataFrame(results).replace({pd.NaT: None, np.nan: None})
-        results = results.to_dict(orient="records")
         query_result.update(
             {
                 "results": frappe.as_json(results),

From 04fa24cedc699645c8bb2f4145a87eb8c6f9d8f3 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Fri, 8 Nov 2024 15:56:40 +0530
Subject: [PATCH 07/15] feat: import a sample workbook for onboarding

---
 insights/setup/demo.py              |  27 +-
 insights/setup/sample_workbook.json | 934 ++++++++++++++++++++++++++++
 2 files changed, 959 insertions(+), 2 deletions(-)
 create mode 100644 insights/setup/sample_workbook.json

diff --git a/insights/setup/demo.py b/insights/setup/demo.py
index a24f3927..dc3e7d00 100644
--- a/insights/setup/demo.py
+++ b/insights/setup/demo.py
@@ -35,7 +35,6 @@ def run(force=False):
             factory = DemoDataFactory()
             factory.initialize()
             if factory.demo_data_exists() and not force:
-                factory.create_table_links()
                 update_progress("Done", 99)
                 return factory
             update_progress("Downloading data...", 5)
@@ -43,6 +42,7 @@ def run(force=False):
             update_progress("Syncing tables...", 60)
             factory.sync_tables()
             update_progress("Done", 99)
+            factory.create_sample_workbook()
             return factory
 
     def initialize(self):
@@ -71,7 +71,18 @@ def setup_demo_data_source(self):
 
     def demo_data_exists(self):
         tables = get_data_source_tables(self.data_source.name)
-        return len(tables) == 8
+        tables_exists = len(tables) == 8
+
+        links_count = frappe.db.count(
+            "Insights Table Link v3", {"data_source": self.data_source.name}
+        )
+        links_exists = links_count == 8
+
+        sample_workbook_exists = frappe.db.exists(
+            "Insights Workbook", {"title": "Order Analysis"}
+        )
+
+        return tables_exists and links_exists and sample_workbook_exists
 
     def download_demo_data(self):
         if frappe.flags.in_test or os.environ.get("CI"):
@@ -159,3 +170,15 @@ def create_table_links(self):
                 link["left_column"],
                 link["right_column"],
             )
+
+    def create_sample_workbook(self):
+        if frappe.db.exists("Insights Workbook", {"title": "Order Analysis"}):
+            return
+
+        fixture_path = frappe.get_app_path("insights", "setup")
+        with open(fixture_path + "/sample_workbook.json") as f:
+            sample_workbook = f.read()
+
+        w = frappe.new_doc("Insights Workbook")
+        w.update(frappe.parse_json(sample_workbook))
+        w.save(ignore_permissions=True)
diff --git a/insights/setup/sample_workbook.json b/insights/setup/sample_workbook.json
new file mode 100644
index 00000000..f0bbd6ea
--- /dev/null
+++ b/insights/setup/sample_workbook.json
@@ -0,0 +1,934 @@
+{
+	"doctype": "Insights Workbook",
+	"title": "Order Analysis",
+	"queries": [
+		{
+			"name": "dmpeh9iw",
+			"operations": [
+				{
+					"table": {
+						"data_source": "demo_data",
+						"table_name": "orders",
+						"type": "table"
+					},
+					"type": "source"
+				},
+				{
+					"join_condition": {
+						"left_column": {
+							"column_name": "order_id",
+							"type": "column"
+						},
+						"right_column": {
+							"column_name": "order_id",
+							"type": "column"
+						}
+					},
+					"join_type": "left",
+					"table": {
+						"data_source": "demo_data",
+						"table_name": "orderitems",
+						"type": "table"
+					},
+					"type": "join"
+				},
+				{
+					"column": {
+						"column_name": "order_purchase_timestamp",
+						"type": "column"
+					},
+					"data_type": "Datetime",
+					"type": "cast"
+				},
+				{
+					"filters": [
+						{
+							"column": {
+								"column_name": "order_status",
+								"type": "column"
+							},
+							"operator": "not_in",
+							"value": [
+								"delivered"
+							]
+						},
+						{
+							"column": {
+								"column_name": "order_id",
+								"type": "column"
+							},
+							"operator": "is_set"
+						},
+						{
+							"column": {
+								"column_name": "customer_id",
+								"type": "column"
+							},
+							"operator": "is_set"
+						},
+						{
+							"expression": {
+								"expression": "(order_purchase_timestamp >= '2016-10-01') & (order_purchase_timestamp <= '2018-08-31')",
+								"type": "expression"
+							}
+						}
+					],
+					"logical_operator": "And",
+					"type": "filter_group"
+				},
+				{
+					"data_type": "Decimal",
+					"expression": {
+						"expression": "q.price + q.freight_value",
+						"type": "expression"
+					},
+					"new_name": "order_value",
+					"type": "mutate"
+				}
+			],
+			"title": "Undelivered Orders",
+			"use_live_connection": true
+		}
+	],
+	"charts": [
+		{
+			"chart_type": "Bar",
+			"config": {
+				"filters": {
+					"filters": [],
+					"logical_operator": "And"
+				},
+				"grouping": "stacked",
+				"limit": "100",
+				"normalize": false,
+				"order_by": [
+					{
+						"column": {
+							"column_name": "order_purchase_timestamp",
+							"type": "column"
+						},
+						"direction": "asc"
+					}
+				],
+				"show_data_labels": false,
+				"split_by": null,
+				"stack": false,
+				"swap_axes": false,
+				"x_axis": {
+					"column_name": "order_purchase_timestamp",
+					"data_type": "Datetime",
+					"granularity": "month",
+					"label": "order_purchase_timestamp",
+					"value": "order_purchase_timestamp"
+				},
+				"y2_axis": null,
+				"y2_axis_type": "line",
+				"y_axis": {
+					"normalize": false,
+					"series": [
+						{
+							"measure": {
+								"aggregation": "sum",
+								"column_name": "price",
+								"data_type": "Decimal",
+								"label": "sum(price)",
+								"measure_name": "sum(price)",
+								"value": "sum(price)"
+							}
+						}
+					],
+					"show_axis_label": false,
+					"show_data_labels": false,
+					"stack": false
+				}
+			},
+			"name": "e2xcyj2w",
+			"operations": [
+				{
+					"table": {
+						"operations": [
+							{
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orders",
+									"type": "table"
+								},
+								"type": "source"
+							},
+							{
+								"join_condition": {
+									"left_column": {
+										"column_name": "order_id",
+										"type": "column"
+									},
+									"right_column": {
+										"column_name": "order_id",
+										"type": "column"
+									}
+								},
+								"join_type": "left",
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orderitems",
+									"type": "table"
+								},
+								"type": "join"
+							},
+							{
+								"column": {
+									"column_name": "order_purchase_timestamp",
+									"type": "column"
+								},
+								"data_type": "Datetime",
+								"type": "cast"
+							},
+							{
+								"filters": [
+									{
+										"column": {
+											"column_name": "order_status",
+											"type": "column"
+										},
+										"operator": "not_in",
+										"value": [
+											"delivered"
+										]
+									},
+									{
+										"column": {
+											"column_name": "order_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"column": {
+											"column_name": "customer_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"expression": {
+											"expression": "(order_purchase_timestamp >= '2016-10-01') & (order_purchase_timestamp <= '2018-08-31')",
+											"type": "expression"
+										}
+									}
+								],
+								"logical_operator": "And",
+								"type": "filter_group"
+							},
+							{
+								"data_type": "Decimal",
+								"expression": {
+									"expression": "q.price + q.freight_value",
+									"type": "expression"
+								},
+								"new_name": "order_value",
+								"type": "mutate"
+							}
+						],
+						"query_name": "dmpeh9iw",
+						"type": "query",
+						"workbook": ""
+					},
+					"type": "source"
+				},
+				{
+					"dimensions": [
+						{
+							"column_name": "order_purchase_timestamp",
+							"data_type": "Datetime",
+							"granularity": "month",
+							"label": "order_purchase_timestamp",
+							"value": "order_purchase_timestamp"
+						}
+					],
+					"measures": [
+						{
+							"aggregation": "sum",
+							"column_name": "price",
+							"data_type": "Decimal",
+							"label": "sum(price)",
+							"measure_name": "sum(price)",
+							"value": "sum(price)"
+						}
+					],
+					"type": "summarize"
+				},
+				{
+					"column": {
+						"column_name": "order_purchase_timestamp",
+						"type": "column"
+					},
+					"direction": "asc",
+					"type": "order_by"
+				},
+				{
+					"limit": "100",
+					"type": "limit"
+				}
+			],
+			"query": "dmpeh9iw",
+			"title": "Order per Month",
+			"use_live_connection": true
+		},
+		{
+			"chart_type": "Donut",
+			"config": {
+				"filters": {
+					"filters": [],
+					"logical_operator": "And"
+				},
+				"label_column": {
+					"column_name": "order_status",
+					"data_type": "String",
+					"label": "order_status",
+					"value": "order_status"
+				},
+				"legend_position": "bottom",
+				"limit": 100,
+				"order_by": [
+					{
+						"column": {
+							"column_name": "sum(order_value)",
+							"type": "column"
+						},
+						"direction": "desc"
+					}
+				],
+				"swap_axes": true,
+				"value_column": {
+					"aggregation": "sum",
+					"column_name": "order_value",
+					"data_type": "Decimal",
+					"label": "sum(order_value)",
+					"measure_name": "sum(order_value)",
+					"value": "sum(order_value)"
+				},
+				"x_axis": {
+					"column_name": "order_status"
+				},
+				"y_axis": {
+					"series": [
+						{
+							"measure": {
+								"column_name": "count"
+							}
+						}
+					]
+				}
+			},
+			"name": "7k43ggsa",
+			"operations": [
+				{
+					"table": {
+						"operations": [
+							{
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orders",
+									"type": "table"
+								},
+								"type": "source"
+							},
+							{
+								"join_condition": {
+									"left_column": {
+										"column_name": "order_id",
+										"type": "column"
+									},
+									"right_column": {
+										"column_name": "order_id",
+										"type": "column"
+									}
+								},
+								"join_type": "left",
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orderitems",
+									"type": "table"
+								},
+								"type": "join"
+							},
+							{
+								"column": {
+									"column_name": "order_purchase_timestamp",
+									"type": "column"
+								},
+								"data_type": "Datetime",
+								"type": "cast"
+							},
+							{
+								"filters": [
+									{
+										"column": {
+											"column_name": "order_status",
+											"type": "column"
+										},
+										"operator": "not_in",
+										"value": [
+											"delivered"
+										]
+									},
+									{
+										"column": {
+											"column_name": "order_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"column": {
+											"column_name": "customer_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"expression": {
+											"expression": "(order_purchase_timestamp >= '2016-10-01') & (order_purchase_timestamp <= '2018-08-31')",
+											"type": "expression"
+										}
+									}
+								],
+								"logical_operator": "And",
+								"type": "filter_group"
+							},
+							{
+								"data_type": "Decimal",
+								"expression": {
+									"expression": "q.price + q.freight_value",
+									"type": "expression"
+								},
+								"new_name": "order_value",
+								"type": "mutate"
+							}
+						],
+						"query_name": "dmpeh9iw",
+						"type": "query",
+						"workbook": ""
+					},
+					"type": "source"
+				},
+				{
+					"dimensions": [
+						{
+							"column_name": "order_status",
+							"data_type": "String",
+							"label": "order_status",
+							"value": "order_status"
+						}
+					],
+					"measures": [
+						{
+							"aggregation": "sum",
+							"column_name": "order_value",
+							"data_type": "Decimal",
+							"label": "sum(order_value)",
+							"measure_name": "sum(order_value)",
+							"value": "sum(order_value)"
+						}
+					],
+					"type": "summarize"
+				},
+				{
+					"column": {
+						"column_name": "sum(order_value)",
+						"type": "column"
+					},
+					"direction": "desc",
+					"type": "order_by"
+				},
+				{
+					"limit": 100,
+					"type": "limit"
+				}
+			],
+			"query": "dmpeh9iw",
+			"title": "Order per Category",
+			"use_live_connection": true
+		},
+		{
+			"chart_type": "Number",
+			"config": {
+				"comparison": true,
+				"date_column": {
+					"column_name": "order_purchase_timestamp",
+					"data_type": "Datetime",
+					"granularity": "month",
+					"label": "order_purchase_timestamp",
+					"value": "order_purchase_timestamp"
+				},
+				"decimal": "2",
+				"filters": {
+					"filters": [],
+					"logical_operator": "And"
+				},
+				"limit": 100,
+				"negative_is_better": false,
+				"number_column": [
+					"price",
+					"freight_value"
+				],
+				"number_columns": [
+					{
+						"aggregation": "count",
+						"column_name": "count",
+						"data_type": "Integer",
+						"label": "count(*)",
+						"measure_name": "Orders",
+						"value": "count(*)"
+					},
+					{
+						"aggregation": "sum",
+						"column_name": "order_item_id",
+						"data_type": "Integer",
+						"label": "sum(order_item_id)",
+						"measure_name": "Items",
+						"value": "sum(order_item_id)"
+					},
+					{
+						"aggregation": "sum",
+						"column_name": "price",
+						"data_type": "Decimal",
+						"label": "sum(price)",
+						"measure_name": "Net Total",
+						"value": "sum(price)"
+					},
+					{
+						"aggregation": "sum",
+						"column_name": "freight_value",
+						"data_type": "Decimal",
+						"label": "sum(freight_value)",
+						"measure_name": "Freight",
+						"value": "sum(freight_value)"
+					},
+					{
+						"aggregation": "sum",
+						"column_name": "order_value",
+						"data_type": "Decimal",
+						"label": "sum(order_value)",
+						"measure_name": "Grand Total",
+						"value": "sum(order_value)"
+					}
+				],
+				"order_by": [
+					{
+						"column": {
+							"column_name": "order_purchase_timestamp",
+							"type": "column"
+						},
+						"direction": "desc"
+					}
+				],
+				"shorten_numbers": true,
+				"sparkline": true
+			},
+			"name": "cdsy4r05",
+			"operations": [
+				{
+					"table": {
+						"operations": [
+							{
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orders",
+									"type": "table"
+								},
+								"type": "source"
+							},
+							{
+								"join_condition": {
+									"left_column": {
+										"column_name": "order_id",
+										"type": "column"
+									},
+									"right_column": {
+										"column_name": "order_id",
+										"type": "column"
+									}
+								},
+								"join_type": "left",
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orderitems",
+									"type": "table"
+								},
+								"type": "join"
+							},
+							{
+								"column": {
+									"column_name": "order_purchase_timestamp",
+									"type": "column"
+								},
+								"data_type": "Datetime",
+								"type": "cast"
+							},
+							{
+								"filters": [
+									{
+										"column": {
+											"column_name": "order_status",
+											"type": "column"
+										},
+										"operator": "not_in",
+										"value": [
+											"delivered"
+										]
+									},
+									{
+										"column": {
+											"column_name": "order_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"column": {
+											"column_name": "customer_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"expression": {
+											"expression": "(order_purchase_timestamp >= '2016-10-01') & (order_purchase_timestamp <= '2018-08-31')",
+											"type": "expression"
+										}
+									}
+								],
+								"logical_operator": "And",
+								"type": "filter_group"
+							},
+							{
+								"data_type": "Decimal",
+								"expression": {
+									"expression": "q.price + q.freight_value",
+									"type": "expression"
+								},
+								"new_name": "order_value",
+								"type": "mutate"
+							}
+						],
+						"query_name": "dmpeh9iw",
+						"type": "query",
+						"workbook": ""
+					},
+					"type": "source"
+				},
+				{
+					"dimensions": [
+						{
+							"column_name": "order_purchase_timestamp",
+							"data_type": "Datetime",
+							"granularity": "month",
+							"label": "order_purchase_timestamp",
+							"value": "order_purchase_timestamp"
+						}
+					],
+					"measures": [
+						{
+							"aggregation": "count",
+							"column_name": "count",
+							"data_type": "Integer",
+							"label": "count(*)",
+							"measure_name": "Orders",
+							"value": "count(*)"
+						},
+						{
+							"aggregation": "sum",
+							"column_name": "order_item_id",
+							"data_type": "Integer",
+							"label": "sum(order_item_id)",
+							"measure_name": "Items",
+							"value": "sum(order_item_id)"
+						},
+						{
+							"aggregation": "sum",
+							"column_name": "price",
+							"data_type": "Decimal",
+							"label": "sum(price)",
+							"measure_name": "Net Total",
+							"value": "sum(price)"
+						},
+						{
+							"aggregation": "sum",
+							"column_name": "freight_value",
+							"data_type": "Decimal",
+							"label": "sum(freight_value)",
+							"measure_name": "Freight",
+							"value": "sum(freight_value)"
+						},
+						{
+							"aggregation": "sum",
+							"column_name": "order_value",
+							"data_type": "Decimal",
+							"label": "sum(order_value)",
+							"measure_name": "Grand Total",
+							"value": "sum(order_value)"
+						}
+					],
+					"type": "summarize"
+				},
+				{
+					"column": {
+						"column_name": "order_purchase_timestamp",
+						"type": "column"
+					},
+					"direction": "desc",
+					"type": "order_by"
+				},
+				{
+					"limit": 100,
+					"type": "limit"
+				}
+			],
+			"query": "dmpeh9iw",
+			"title": "Top Level Metrics",
+			"use_live_connection": true
+		},
+		{
+			"chart_type": "Table",
+			"config": {
+				"columns": [
+					{
+						"column_name": "order_status",
+						"data_type": "String",
+						"label": "order_status",
+						"value": "order_status"
+					}
+				],
+				"filters": {
+					"filters": [],
+					"logical_operator": "And"
+				},
+				"limit": 100,
+				"order_by": [
+					{
+						"column": {
+							"column_name": "order_purchase_timestamp",
+							"type": "column"
+						},
+						"direction": "asc"
+					}
+				],
+				"rows": [
+					{
+						"column_name": "order_purchase_timestamp",
+						"data_type": "Datetime",
+						"granularity": "month",
+						"label": "order_purchase_timestamp",
+						"value": "order_purchase_timestamp"
+					}
+				],
+				"show_column_totals": false,
+				"show_row_totals": false,
+				"values": [
+					{
+						"aggregation": "sum",
+						"column_name": "order_value",
+						"data_type": "Decimal",
+						"label": "sum(order_value)",
+						"measure_name": "sum(order_value)",
+						"value": "sum(order_value)"
+					}
+				]
+			},
+			"name": "9p7rtr1u",
+			"operations": [
+				{
+					"table": {
+						"operations": [
+							{
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orders",
+									"type": "table"
+								},
+								"type": "source"
+							},
+							{
+								"join_condition": {
+									"left_column": {
+										"column_name": "order_id",
+										"type": "column"
+									},
+									"right_column": {
+										"column_name": "order_id",
+										"type": "column"
+									}
+								},
+								"join_type": "left",
+								"table": {
+									"data_source": "demo_data",
+									"table_name": "orderitems",
+									"type": "table"
+								},
+								"type": "join"
+							},
+							{
+								"column": {
+									"column_name": "order_purchase_timestamp",
+									"type": "column"
+								},
+								"data_type": "Datetime",
+								"type": "cast"
+							},
+							{
+								"filters": [
+									{
+										"column": {
+											"column_name": "order_status",
+											"type": "column"
+										},
+										"operator": "not_in",
+										"value": [
+											"delivered"
+										]
+									},
+									{
+										"column": {
+											"column_name": "order_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"column": {
+											"column_name": "customer_id",
+											"type": "column"
+										},
+										"operator": "is_set"
+									},
+									{
+										"expression": {
+											"expression": "(order_purchase_timestamp >= '2016-10-01') & (order_purchase_timestamp <= '2018-08-31')",
+											"type": "expression"
+										}
+									}
+								],
+								"logical_operator": "And",
+								"type": "filter_group"
+							},
+							{
+								"data_type": "Decimal",
+								"expression": {
+									"expression": "q.price + q.freight_value",
+									"type": "expression"
+								},
+								"new_name": "order_value",
+								"type": "mutate"
+							}
+						],
+						"query_name": "dmpeh9iw",
+						"type": "query",
+						"workbook": ""
+					},
+					"type": "source"
+				},
+				{
+					"columns": [
+						{
+							"column_name": "order_status",
+							"data_type": "String",
+							"label": "order_status",
+							"value": "order_status"
+						}
+					],
+					"rows": [
+						{
+							"column_name": "order_purchase_timestamp",
+							"data_type": "Datetime",
+							"granularity": "month",
+							"label": "order_purchase_timestamp",
+							"value": "order_purchase_timestamp"
+						}
+					],
+					"type": "pivot_wider",
+					"values": [
+						{
+							"aggregation": "sum",
+							"column_name": "order_value",
+							"data_type": "Decimal",
+							"label": "sum(order_value)",
+							"measure_name": "sum(order_value)",
+							"value": "sum(order_value)"
+						}
+					]
+				},
+				{
+					"column": {
+						"column_name": "order_purchase_timestamp",
+						"type": "column"
+					},
+					"direction": "asc",
+					"type": "order_by"
+				},
+				{
+					"limit": 100,
+					"type": "limit"
+				}
+			],
+			"query": "dmpeh9iw",
+			"title": "Order Value per Month per Status",
+			"use_live_connection": true
+		}
+	],
+	"dashboards": [
+		{
+			"items": [
+				{
+					"chart": "cdsy4r05",
+					"layout": {
+						"h": 3,
+						"i": "1geec587",
+						"moved": false,
+						"w": 20,
+						"x": 0,
+						"y": 0
+					},
+					"type": "chart"
+				},
+				{
+					"chart": "e2xcyj2w",
+					"layout": {
+						"h": 8,
+						"i": "1kpzltkq",
+						"moved": false,
+						"w": 10,
+						"x": 0,
+						"y": 3
+					},
+					"type": "chart"
+				},
+				{
+					"chart": "7k43ggsa",
+					"layout": {
+						"h": 8,
+						"i": "2mkbxvt3",
+						"moved": false,
+						"w": 10,
+						"x": 10,
+						"y": 3
+					},
+					"type": "chart"
+				},
+				{
+					"chart": "9p7rtr1u",
+					"layout": {
+						"h": 11,
+						"i": "b6n952s4",
+						"moved": false,
+						"w": 20,
+						"x": 0,
+						"y": 11
+					},
+					"type": "chart"
+				}
+			],
+			"name": "do859say",
+			"title": "Order Analysis"
+		}
+	]
+}

From 462b8493a11eabb6dd1d15a1218065d7a3932cf5 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Fri, 8 Nov 2024 16:34:35 +0530
Subject: [PATCH 08/15] chore: ui changes

---
 frontend/src2/components/DataTable.vue        | 30 +++++++---
 .../query/components/AddOperationPopover.vue  | 56 ++++++++++---------
 .../query/components/QueryBuilderToolbar.vue  | 51 ++++++++---------
 frontend/src2/query/query.ts                  |  2 +
 frontend/src2/types/query.types.ts            |  1 +
 insights/api/data_sources.py                  |  2 +-
 insights/api/workbooks.py                     | 11 ++--
 .../insights_data_source_v3/ibis_utils.py     |  7 ++-
 8 files changed, 93 insertions(+), 67 deletions(-)

diff --git a/frontend/src2/components/DataTable.vue b/frontend/src2/components/DataTable.vue
index 88e66cf3..ee067b3c 100644
--- a/frontend/src2/components/DataTable.vue
+++ b/frontend/src2/components/DataTable.vue
@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { Download, Table2Icon } from 'lucide-vue-next'
+import { Download, Search, Table2Icon } from 'lucide-vue-next'
 import { computed, ref } from 'vue'
 import { formatNumber } from '../helpers'
 import { FIELDTYPES } from '../helpers/constants'
@@ -106,9 +106,12 @@ const filterPerColumn = ref<Record<string, string>>({})
 	>
 		<div class="w-full flex-1 overflow-y-auto">
 			<table class="relative h-full w-full border-separate border-spacing-0">
-				<thead class="sticky top-0 z-10 bg-white">
+				<thead class="sticky top-0 z-10 bg-gray-50">
 					<tr>
-						<td class="whitespace-nowrap border-b border-r" width="1%"></td>
+						<td
+							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-gray-50"
+							width="1%"
+						></td>
 						<td
 							v-for="(column, idx) in props.columns"
 							:key="idx"
@@ -130,15 +133,26 @@ const filterPerColumn = ref<Record<string, string>>({})
 							<div class="truncate pl-3 pr-20"></div>
 						</td>
 					</tr>
-
 					<tr v-if="props.showFilterRow">
-						<td class="whitespace-nowrap border-b border-r" width="1%"></td>
+						<td
+							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-gray-50"
+							width="1%"
+						></td>
 						<td
 							v-for="(column, idx) in props.columns"
 							:key="idx"
-							class="z-0 border-b border-r p-1"
+							class="border-b border-r p-0.5"
 						>
-							<FormControl type="text" v-model="filterPerColumn[column.name]" />
+							<FormControl
+								type="text"
+								v-model="filterPerColumn[column.name]"
+								autocomplete="off"
+								class="[&_input]:bg-gray-200/80"
+							>
+								<template #prefix>
+									<Search class="h-4 w-4 text-gray-500" stroke-width="1.5" />
+								</template>
+							</FormControl>
 						</td>
 						<td
 							v-if="props.showRowTotals"
@@ -152,7 +166,7 @@ const filterPerColumn = ref<Record<string, string>>({})
 				<tbody>
 					<tr v-for="(row, idx) in visibleRows?.slice(0, 100)" :key="idx">
 						<td
-							class="whitespace-nowrap border-b border-r px-3 text-right"
+							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-gray-50 px-3 text-right"
 							width="1%"
 							height="30px"
 						>
diff --git a/frontend/src2/query/components/AddOperationPopover.vue b/frontend/src2/query/components/AddOperationPopover.vue
index f5d71be3..d2ec1426 100644
--- a/frontend/src2/query/components/AddOperationPopover.vue
+++ b/frontend/src2/query/components/AddOperationPopover.vue
@@ -56,7 +56,7 @@ const operationButtons = [
 	},
 	{
 		label: 'Filter Rows',
-		description: 'Filter rows based on columns or custom expressions',
+		description: 'Filter rows based on columns or expressions',
 		icon: query_operation_types.filter.icon,
 		onClick: () => (showFiltersSelectorDialog.value = true),
 	},
@@ -138,32 +138,36 @@ watch(
 				<span class="flex h-6 items-center px-2 text-xs text-gray-500">
 					Select an operation
 				</span>
-				<Button
-					v-for="button in operationButtons"
-					:key="button.label"
-					variant="ghost"
-					class="!h-fit w-full !justify-start"
-					@click="
-						() => {
-							button.onClick()
-							togglePopover()
-						}
-					"
-				>
-					<template #prefix>
-						<div class="flex items-start gap-2 py-2">
-							<component
-								:is="button.icon"
-								class="h-4.5 w-4.5 flex-shrink-0 text-gray-700"
-								stroke-width="1.5"
-							/>
-							<div class="flex flex-1 flex-col gap-1 text-left">
-								<p class="truncate text-sm">{{ button.label }}</p>
-								<p class="w-40 text-xs text-gray-500">{{ button.description }}</p>
+				<div class="grid grid-cols-2">
+					<Button
+						v-for="button in operationButtons"
+						:key="button.label"
+						variant="ghost"
+						class="!h-fit w-full !justify-start"
+						@click="
+							() => {
+								button.onClick()
+								togglePopover()
+							}
+						"
+					>
+						<template #prefix>
+							<div class="flex items-start gap-2 py-2">
+								<component
+									:is="button.icon"
+									class="h-4.5 w-4.5 flex-shrink-0 text-gray-700"
+									stroke-width="1.5"
+								/>
+								<div class="flex flex-1 flex-col gap-1 text-left">
+									<p class="truncate text-sm">{{ button.label }}</p>
+									<p class="w-40 text-xs text-gray-500">
+										{{ button.description }}
+									</p>
+								</div>
 							</div>
-						</div>
-					</template>
-				</Button>
+						</template>
+					</Button>
+				</div>
 			</div>
 		</template>
 	</Popover>
diff --git a/frontend/src2/query/components/QueryBuilderToolbar.vue b/frontend/src2/query/components/QueryBuilderToolbar.vue
index 4c084dcd..da0e1909 100644
--- a/frontend/src2/query/components/QueryBuilderToolbar.vue
+++ b/frontend/src2/query/components/QueryBuilderToolbar.vue
@@ -1,8 +1,6 @@
 <script setup lang="ts">
-import { Tooltip } from 'frappe-ui'
-import { ArrowLeftRight, CodeIcon, Columns, Download, PlayIcon } from 'lucide-vue-next'
+import { CodeIcon, PlayIcon, Scroll } from 'lucide-vue-next'
 import { inject, ref } from 'vue'
-import ContentEditable from '../../components/ContentEditable.vue'
 import { Query } from '../query'
 import ViewSQLDialog from './ViewSQLDialog.vue'
 
@@ -11,16 +9,13 @@ const query = inject('query') as Query
 const showViewSQLDialog = ref(false)
 
 const actions = [
-	{
-		type: 'separator',
-	},
 	// {
 	// 	label: 'Manage Columns',
 	// 	icon: Columns,
 	// },
 	{
 		label: 'View SQL',
-		icon: CodeIcon,
+		icon: Scroll,
 		onClick: () => (showViewSQLDialog.value = true),
 	},
 	{
@@ -32,25 +27,31 @@ const actions = [
 </script>
 
 <template>
-	<div class="flex w-full flex-shrink-0 items-center justify-between bg-gray-50 p-2">
-		<div class="flex w-full items-center gap-2">
+	<div class="flex w-full flex-shrink-0 items-center justify-between bg-white p-2">
+		<div
+			v-if="query.result.executedSQL"
+			class="tnum flex items-center gap-2 px-2 text-sm text-gray-600"
+		>
+			<div class="h-2 w-2 rounded-full bg-green-500"></div>
+			<span v-if="query.result.timeTaken == -1"> Fetched from cache </span>
+			<span v-else> Fetched in {{ query.result.timeTaken }} ms </span>
+		</div>
+		<div class="flex items-center gap-2">
 			<template v-for="(action, idx) in actions" :key="idx">
-				<div v-if="action.type === 'separator'" class="h-7 flex-1"></div>
-				<Tooltip v-else placement="top" :hover-delay="0.1" :text="action.label">
-					<Button
-						:variant="'ghost'"
-						@click="action.onClick"
-						class="h-7 w-7 bg-white shadow"
-					>
-						<template #icon>
-							<component
-								:is="action.icon"
-								class="h-4 w-4 text-gray-700"
-								stroke-width="1.5"
-							/>
-						</template>
-					</Button>
-				</Tooltip>
+				<Button
+					:variant="'ghost'"
+					:label="action.label"
+					@click="action.onClick"
+					class="bg-white text-sm shadow"
+				>
+					<template #prefix>
+						<component
+							:is="action.icon"
+							class="h-3.5 w-3.5 text-gray-700"
+							stroke-width="1.5"
+						/>
+					</template>
+				</Button>
 			</template>
 		</div>
 	</div>
diff --git a/frontend/src2/query/query.ts b/frontend/src2/query/query.ts
index 4e47a703..e3722212 100644
--- a/frontend/src2/query/query.ts
+++ b/frontend/src2/query/query.ts
@@ -255,6 +255,7 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 					query: query.doc.name,
 					data_type: column.type,
 				}))
+				query.result.timeTaken = response.time_taken
 			})
 			.catch((e: Error) => {
 				query.result = { ...EMPTY_RESULT }
@@ -718,6 +719,7 @@ const EMPTY_RESULT = {
 	formattedRows: [],
 	columns: [],
 	columnOptions: [],
+	timeTaken: 0,
 } as QueryResult
 
 export type Query = ReturnType<typeof makeQuery>
diff --git a/frontend/src2/types/query.types.ts b/frontend/src2/types/query.types.ts
index 79e49768..88327392 100644
--- a/frontend/src2/types/query.types.ts
+++ b/frontend/src2/types/query.types.ts
@@ -190,4 +190,5 @@ export type QueryResult = {
 	formattedRows: QueryResultRow[]
 	columns: QueryResultColumn[]
 	columnOptions: ColumnOption[]
+	timeTaken: number
 }
diff --git a/insights/api/data_sources.py b/insights/api/data_sources.py
index 5ce3b13b..09c8881c 100644
--- a/insights/api/data_sources.py
+++ b/insights/api/data_sources.py
@@ -335,7 +335,7 @@ def get_data_source_table(data_source: str, table_name: str):
     ds = frappe.get_doc("Insights Data Source v3", data_source)
     db = ds._get_ibis_backend()
     q = db.table(table_name).head(100)
-    data = execute_ibis_query(q, cache_expiry=24 * 60 * 60)
+    data, time_taken = execute_ibis_query(q, cache_expiry=24 * 60 * 60)
 
     return {
         "table_name": table_name,
diff --git a/insights/api/workbooks.py b/insights/api/workbooks.py
index 7582bc19..b3cc8bb4 100644
--- a/insights/api/workbooks.py
+++ b/insights/api/workbooks.py
@@ -19,13 +19,16 @@ def fetch_query_results(operations, limit=100, use_live_connection=True):
         return
 
     columns = get_columns_from_schema(ibis_query.schema())
-    results = execute_ibis_query(ibis_query, limit=limit, cache_expiry=60 * 10)
+    results, time_taken = execute_ibis_query(
+        ibis_query, limit=limit, cache_expiry=60 * 10
+    )
     results = results.to_dict(orient="records")
 
     return {
         "sql": ibis.to_sql(ibis_query),
         "columns": columns,
         "rows": results,
+        "time_taken": time_taken,
     }
 
 
@@ -36,7 +39,7 @@ def fetch_query_results_count(operations, use_live_connection=True):
         return
 
     count_query = ibis_query.aggregate(count=_.count())
-    count_results = execute_ibis_query(count_query, cache_expiry=60 * 5)
+    count_results, time_taken = execute_ibis_query(count_query, cache_expiry=60 * 5)
     total_count = count_results.values[0][0]
     return int(total_count)
 
@@ -47,7 +50,7 @@ def download_query_results(operations, use_live_connection=True):
     if ibis_query is None:
         return
 
-    results = execute_ibis_query(ibis_query, cache=False, limit=10_00_000)
+    results, time_taken = execute_ibis_query(ibis_query, cache=False, limit=10_00_000)
     return results.to_csv(index=False)
 
 
@@ -66,7 +69,7 @@ def get_distinct_column_values(
         .distinct()
         .head(limit)
     )
-    result = execute_ibis_query(values_query, cache_expiry=24 * 60 * 60)
+    result, time_taken = execute_ibis_query(values_query, cache_expiry=24 * 60 * 60)
     return result[column_name].tolist()
 
 
diff --git a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
index ee05ffc5..a9946df9 100644
--- a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
+++ b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
@@ -507,18 +507,19 @@ def execute_ibis_query(
     sql = ibis.to_sql(query)
 
     if cache and has_cached_results(sql):
-        return get_cached_results(sql)
+        return get_cached_results(sql), -1
 
     start = time.monotonic()
     res: pd.DataFrame = query.execute()
-    create_execution_log(sql, flt(time.monotonic() - start, 3))
+    time_taken = flt(time.monotonic() - start, 3)
+    create_execution_log(sql, time_taken)
 
     res = res.replace({pd.NaT: None, np.nan: None})
 
     if cache:
         cache_results(sql, res, cache_expiry)
 
-    return res
+    return res, time_taken
 
 
 def get_columns_from_schema(schema: ibis.Schema):

From 4653cac2cb67f965f8040ce71cff29fab14be339 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Fri, 8 Nov 2024 19:41:01 +0530
Subject: [PATCH 09/15] refactor(ui): remove uncessary borders

---
 frontend/src2/charts/ChartBuilder.vue                  |  6 +++---
 frontend/src2/charts/components/ChartBuilderTable.vue  |  4 ++--
 frontend/src2/charts/components/CollapsibleSection.vue |  2 +-
 frontend/src2/components/DataTable.vue                 | 10 +++++-----
 frontend/src2/dashboard/DashboardBuilder.vue           |  6 +++---
 frontend/src2/query/QueryBuilder.vue                   |  8 +++-----
 frontend/src2/query/components/QueryBuilderTable.vue   |  2 +-
 frontend/src2/query/components/QueryBuilderToolbar.vue | 10 +++++-----
 frontend/src2/query/components/QueryInfo.vue           |  4 ++--
 frontend/src2/workbook/Workbook.vue                    |  2 +-
 frontend/src2/workbook/WorkbookSidebar.vue             |  2 +-
 frontend/src2/workbook/WorkbookSidebarListSection.vue  |  4 ++--
 12 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/frontend/src2/charts/ChartBuilder.vue b/frontend/src2/charts/ChartBuilder.vue
index 9ba157d0..bfb9ec32 100644
--- a/frontend/src2/charts/ChartBuilder.vue
+++ b/frontend/src2/charts/ChartBuilder.vue
@@ -68,11 +68,11 @@ const showShareDialog = ref(false)
 
 <template>
 	<div v-if="chart" class="relative flex h-full w-full overflow-hidden">
-		<div class="relative flex h-full w-full flex-col overflow-hidden">
+		<div class="relative flex h-full w-full flex-col gap-4 overflow-hidden p-3 pt-4">
 			<LoadingOverlay v-if="chart.dataQuery.executing" />
 			<div
 				ref="chartEl"
-				class="flex min-h-[24rem] flex-1 flex-shrink-0 items-center justify-center overflow-hidden p-4"
+				class="flex min-h-[24rem] flex-1 flex-shrink-0 items-center justify-center"
 			>
 				<ChartRenderer
 					:title="chart.doc.title"
@@ -87,7 +87,7 @@ const showShareDialog = ref(false)
 			<ChartBuilderTable v-if="chart.dataQuery.result.executedSQL" />
 		</div>
 		<div
-			class="relative z-[1] flex w-[19rem] flex-shrink-0 flex-col divide-y overflow-y-auto bg-white shadow"
+			class="relative z-[1] flex w-[19rem] flex-shrink-0 flex-col divide-y overflow-y-auto bg-white px-3.5"
 		>
 			<CollapsibleSection title="Chart">
 				<div class="flex flex-col gap-3">
diff --git a/frontend/src2/charts/components/ChartBuilderTable.vue b/frontend/src2/charts/components/ChartBuilderTable.vue
index 4ad34208..38734ac0 100644
--- a/frontend/src2/charts/components/ChartBuilderTable.vue
+++ b/frontend/src2/charts/components/ChartBuilderTable.vue
@@ -12,9 +12,9 @@ const drillOn = ref<{ row: QueryResultRow; column: QueryResultColumn } | null>(n
 </script>
 
 <template>
-	<div v-if="chart.doc.chart_type != 'Table'" class="flex h-[18rem] flex-col divide-y border">
+	<div v-if="chart.doc.chart_type != 'Table'" class="flex h-[18rem] flex-col">
 		<DataTable
-			class="bg-white"
+			class="rounded border bg-white"
 			:columns="chart.dataQuery.result.columns"
 			:rows="chart.dataQuery.result.formattedRows"
 			:on-export="chart.dataQuery.downloadResults"
diff --git a/frontend/src2/charts/components/CollapsibleSection.vue b/frontend/src2/charts/components/CollapsibleSection.vue
index 2c7fe25d..8c0fb302 100644
--- a/frontend/src2/charts/components/CollapsibleSection.vue
+++ b/frontend/src2/charts/components/CollapsibleSection.vue
@@ -8,7 +8,7 @@ const collapsed = ref(props.collapsed ?? false)
 </script>
 
 <template>
-	<div class="flex flex-col px-3.5" :class="collapsed ? '' : 'pb-3.5'">
+	<div class="flex flex-col" :class="collapsed ? '' : 'pb-3.5'">
 		<button
 			class="sticky top-0 flex cursor-pointer items-center gap-1 bg-white py-3"
 			@click="collapsed = !collapsed"
diff --git a/frontend/src2/components/DataTable.vue b/frontend/src2/components/DataTable.vue
index ee067b3c..10d7f5cb 100644
--- a/frontend/src2/components/DataTable.vue
+++ b/frontend/src2/components/DataTable.vue
@@ -106,10 +106,10 @@ const filterPerColumn = ref<Record<string, string>>({})
 	>
 		<div class="w-full flex-1 overflow-y-auto">
 			<table class="relative h-full w-full border-separate border-spacing-0">
-				<thead class="sticky top-0 z-10 bg-gray-50">
+				<thead class="sticky top-0 z-10 bg-white">
 					<tr>
 						<td
-							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-gray-50"
+							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-white"
 							width="1%"
 						></td>
 						<td
@@ -135,7 +135,7 @@ const filterPerColumn = ref<Record<string, string>>({})
 					</tr>
 					<tr v-if="props.showFilterRow">
 						<td
-							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-gray-50"
+							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-white"
 							width="1%"
 						></td>
 						<td
@@ -166,7 +166,7 @@ const filterPerColumn = ref<Record<string, string>>({})
 				<tbody>
 					<tr v-for="(row, idx) in visibleRows?.slice(0, 100)" :key="idx">
 						<td
-							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-gray-50 px-3 text-right"
+							class="sticky left-0 z-10 whitespace-nowrap border-b border-r bg-white px-2 text-right text-xs"
 							width="1%"
 							height="30px"
 						>
@@ -237,7 +237,7 @@ const filterPerColumn = ref<Record<string, string>>({})
 
 	<div
 		v-else-if="props.loading"
-		class="absolute top-10 z-10 flex h-[calc(100%-2rem)] w-full items-center justify-center rounded bg-gray-50/30 backdrop-blur-sm"
+		class="absolute top-10 z-10 flex h-[calc(100%-2rem)] w-full items-center justify-center rounded bg-white/30 backdrop-blur-sm"
 	>
 		<LoadingIndicator class="h-8 w-8 text-gray-700" />
 	</div>
diff --git a/frontend/src2/dashboard/DashboardBuilder.vue b/frontend/src2/dashboard/DashboardBuilder.vue
index 8765f879..652804ab 100644
--- a/frontend/src2/dashboard/DashboardBuilder.vue
+++ b/frontend/src2/dashboard/DashboardBuilder.vue
@@ -59,9 +59,9 @@ const showShareDialog = ref(false)
 </script>
 
 <template>
-	<div class="relative flex h-full w-full divide-x overflow-hidden">
+	<div class="relative flex h-full w-full overflow-hidden bg-gray-50">
 		<div class="relative flex h-full w-full flex-col overflow-hidden">
-			<div class="flex items-center justify-between border-x px-4 pt-3">
+			<div class="flex items-center justify-between p-4 pb-3">
 				<ContentEditable
 					class="cursor-text rounded-sm text-lg font-semibold !text-gray-800 focus:ring-2 focus:ring-gray-700 focus:ring-offset-4"
 					v-model="dashboard.doc.title"
@@ -122,7 +122,7 @@ const showShareDialog = ref(false)
 					</Button>
 				</div>
 			</div>
-			<div class="flex-1 overflow-y-auto p-2" @dragover="onDragOver" @drop="onDrop">
+			<div class="flex-1 overflow-y-auto p-2 pt-0" @dragover="onDragOver" @drop="onDrop">
 				<VueGridLayout
 					v-if="dashboard.doc.items.length > 0"
 					class="h-fit w-full"
diff --git a/frontend/src2/query/QueryBuilder.vue b/frontend/src2/query/QueryBuilder.vue
index 8e03be52..865e7687 100644
--- a/frontend/src2/query/QueryBuilder.vue
+++ b/frontend/src2/query/QueryBuilder.vue
@@ -30,14 +30,12 @@ onBeforeUnmount(() => {
 <template>
 	<div class="flex flex-1 overflow-hidden">
 		<QueryBuilderSourceSelector v-if="!query.doc.operations.length" />
-		<div v-else class="relative flex h-full w-full flex-col overflow-hidden">
+		<div v-else class="relative flex h-full w-full flex-col gap-3 overflow-hidden p-3">
 			<QueryBuilderToolbar></QueryBuilderToolbar>
-			<div class="flex flex-1 overflow-hidden border">
-				<QueryBuilderTable></QueryBuilderTable>
-			</div>
+			<QueryBuilderTable></QueryBuilderTable>
 		</div>
 		<div
-			class="relative z-[1] flex h-full w-[19rem] flex-shrink-0 flex-col divide-y overflow-y-auto bg-white shadow"
+			class="relative z-[1] flex h-full w-[19rem] flex-shrink-0 flex-col overflow-y-auto bg-white"
 		>
 			<QueryInfo />
 			<QueryOperations />
diff --git a/frontend/src2/query/components/QueryBuilderTable.vue b/frontend/src2/query/components/QueryBuilderTable.vue
index 5cce4689..26db4b9a 100644
--- a/frontend/src2/query/components/QueryBuilderTable.vue
+++ b/frontend/src2/query/components/QueryBuilderTable.vue
@@ -17,7 +17,7 @@ const totalRowCount = computed(() =>
 </script>
 
 <template>
-	<div class="relative flex w-full flex-1 flex-col overflow-hidden bg-white">
+	<div class="relative flex w-full flex-1 flex-col overflow-hidden rounded shadow">
 		<div
 			v-if="query.executing"
 			class="absolute top-10 z-10 flex h-[calc(100%-2rem)] w-full items-center justify-center rounded bg-gray-50/30 backdrop-blur-sm"
diff --git a/frontend/src2/query/components/QueryBuilderToolbar.vue b/frontend/src2/query/components/QueryBuilderToolbar.vue
index da0e1909..c2dafe10 100644
--- a/frontend/src2/query/components/QueryBuilderToolbar.vue
+++ b/frontend/src2/query/components/QueryBuilderToolbar.vue
@@ -27,10 +27,10 @@ const actions = [
 </script>
 
 <template>
-	<div class="flex w-full flex-shrink-0 items-center justify-between bg-white p-2">
+	<div class="flex w-full flex-shrink-0 items-center justify-between bg-white">
 		<div
-			v-if="query.result.executedSQL"
-			class="tnum flex items-center gap-2 px-2 text-sm text-gray-600"
+			v-show="query.result.executedSQL"
+			class="tnum flex items-center gap-2 text-sm text-gray-600"
 		>
 			<div class="h-2 w-2 rounded-full bg-green-500"></div>
 			<span v-if="query.result.timeTaken == -1"> Fetched from cache </span>
@@ -42,12 +42,12 @@ const actions = [
 					:variant="'ghost'"
 					:label="action.label"
 					@click="action.onClick"
-					class="bg-white text-sm shadow"
+					class="!h-6 !gap-1.5 bg-white !px-2 text-xs shadow"
 				>
 					<template #prefix>
 						<component
 							:is="action.icon"
-							class="h-3.5 w-3.5 text-gray-700"
+							class="h-3 w-3 text-gray-700"
 							stroke-width="1.5"
 						/>
 					</template>
diff --git a/frontend/src2/query/components/QueryInfo.vue b/frontend/src2/query/components/QueryInfo.vue
index 9d015555..3fa59bb8 100644
--- a/frontend/src2/query/components/QueryInfo.vue
+++ b/frontend/src2/query/components/QueryInfo.vue
@@ -22,14 +22,14 @@ function toggleLiveConnection(enable: boolean) {
 </script>
 
 <template>
-	<div class="flex flex-col px-3.5 py-3">
+	<div class="flex flex-col px-3.5 pt-3">
 		<div class="mb-1 flex h-6 items-center justify-between">
 			<div class="flex items-center gap-1">
 				<div class="text-sm font-medium">Details</div>
 			</div>
 			<div></div>
 		</div>
-		<div class="flex flex-shrink-0 flex-col gap-2.5 px-0.5">
+		<div class="flex flex-shrink-0 flex-col gap-2.5 border-b px-0.5 pb-3">
 			<InlineFormControlLabel label="Query Title">
 				<FormControl v-model="query.doc.title" autocomplete="off" placeholder="Title" />
 			</InlineFormControlLabel>
diff --git a/frontend/src2/workbook/Workbook.vue b/frontend/src2/workbook/Workbook.vue
index e4e288dc..eb334178 100644
--- a/frontend/src2/workbook/Workbook.vue
+++ b/frontend/src2/workbook/Workbook.vue
@@ -53,7 +53,7 @@ watchEffect(() => {
 	<div class="flex h-full w-full flex-col">
 		<WorkbookNavbar />
 		<div
-			class="relative flex w-full flex-1 overflow-hidden bg-gray-50"
+			class="relative flex w-full flex-1 overflow-hidden"
 			:class="workbook.showSidebar ? 'flex-row' : 'flex-col'"
 		>
 			<WorkbookSidebar v-if="workbook.showSidebar" />
diff --git a/frontend/src2/workbook/WorkbookSidebar.vue b/frontend/src2/workbook/WorkbookSidebar.vue
index 323ccd72..f24e170a 100644
--- a/frontend/src2/workbook/WorkbookSidebar.vue
+++ b/frontend/src2/workbook/WorkbookSidebar.vue
@@ -11,7 +11,7 @@ const workbook = inject(workbookKey) as Workbook
 <template>
 	<div
 		v-if="workbook"
-		class="relative z-[1] flex h-full w-[17rem] flex-shrink-0 flex-col overflow-y-auto bg-white shadow"
+		class="relative z-[1] flex h-full w-[17rem] flex-shrink-0 flex-col overflow-y-auto bg-white"
 	>
 		<WorkbookSidebarListSection
 			v-bind="{
diff --git a/frontend/src2/workbook/WorkbookSidebarListSection.vue b/frontend/src2/workbook/WorkbookSidebarListSection.vue
index e133be94..8a1ecce8 100644
--- a/frontend/src2/workbook/WorkbookSidebarListSection.vue
+++ b/frontend/src2/workbook/WorkbookSidebarListSection.vue
@@ -19,7 +19,7 @@ function setDraggedItem(event: DragEvent, row: any) {
 </script>
 
 <template>
-	<div class="flex flex-col border-b px-3.5 py-3">
+	<div class="flex flex-col px-3.5 pt-3">
 		<div class="mb-1 flex h-6 items-center justify-between">
 			<div class="flex items-center gap-1">
 				<div class="text-sm font-medium">{{ section.title }}</div>
@@ -39,7 +39,7 @@ function setDraggedItem(event: DragEvent, row: any) {
 		>
 			<div class="text-xs text-gray-500">{{ section.emptyMessage }}</div>
 		</div>
-		<div v-else class="flex flex-col">
+		<div v-else class="flex flex-col border-b pb-3">
 			<div
 				v-for="(row, idx) in section.items"
 				:key="row[section.itemKey]"

From b85a1a345e31cc27a6b4cf578411faa99fd9d4c0 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Fri, 8 Nov 2024 19:48:38 +0530
Subject: [PATCH 10/15] chore: minor style fix

---
 frontend/src/query/visual/ColumnSection.vue | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/frontend/src/query/visual/ColumnSection.vue b/frontend/src/query/visual/ColumnSection.vue
index 4ff2b412..1fd40416 100644
--- a/frontend/src/query/visual/ColumnSection.vue
+++ b/frontend/src/query/visual/ColumnSection.vue
@@ -110,13 +110,14 @@ function onColumnSort(e) {
 			:showHandle="false"
 		>
 			<template #item="{ item: column, index: idx }">
-				<ColumnListItem
-					v-if="isExpressionColumn(column)"
-					:column="column"
-					:isActive="activeColumnIdx === idx"
-					@edit-column="activeColumnIdx = idx"
-					@remove-column="assistedQuery.removeColumnAt(idx)"
-				/>
+				<div v-if="isExpressionColumn(column)">
+					<ColumnListItem
+						:column="column"
+						:isActive="activeColumnIdx === idx"
+						@edit-column="activeColumnIdx = idx"
+						@remove-column="assistedQuery.removeColumnAt(idx)"
+					/>
+				</div>
 				<Popover
 					v-else
 					:show="showSimpleColumnEditor && activeColumnIdx === idx"

From 47c9b3223964ccd27d9d194b83c23dba7d438098 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Sat, 9 Nov 2024 12:44:06 +0530
Subject: [PATCH 11/15] fix: various fixes

---
 .../query/components/JoinSelectorDialog.vue   | 12 +++----
 .../query/components/QueryBuilderToolbar.vue  | 18 +++++-----
 frontend/src2/query/query.ts                  | 22 ++++++++----
 .../insights_data_source_v3/ibis_utils.py     | 36 +++++++++++--------
 4 files changed, 52 insertions(+), 36 deletions(-)

diff --git a/frontend/src2/query/components/JoinSelectorDialog.vue b/frontend/src2/query/components/JoinSelectorDialog.vue
index 4b0e4462..146912c4 100644
--- a/frontend/src2/query/components/JoinSelectorDialog.vue
+++ b/frontend/src2/query/components/JoinSelectorDialog.vue
@@ -34,7 +34,7 @@ const join = reactive<JoinArgs>(
 				select_columns: [],
 		  }
 )
-const selectedTableOption = computed({
+const selectedTable = computed({
 	get() {
 		if (join.table.type === 'table' && join.table.table_name) {
 			return `${join.table.data_source}.${join.table.table_name}`
@@ -65,13 +65,11 @@ const data_source = computed(() => {
 	// TODO: support multiple data source joins if live connection is disabled
 	// if (!query.doc.use_live_connection) return undefined
 
-	const operations = query.getOperationsForExecution()
-	const source = operations.find((op) => op.type === 'source')
-	return source && source.table.type === 'table' ? source.table.data_source : ''
+	return query.source
 })
 
-wheneverChanges(selectedTableOption, () => {
-	if (!selectedTableOption.value) return
+wheneverChanges(selectedTable, () => {
+	if (!selectedTable.value) return
 
 	// reset previous values if table is changed
 	join.select_columns = []
@@ -250,7 +248,7 @@ function reset() {
 						<label class="mb-1 block text-xs text-gray-600">Select Table to Join</label>
 						<Autocomplete
 							placeholder="Table"
-							v-model="selectedTableOption"
+							v-model="selectedTable"
 							:loading="tableOptions.loading"
 							:options="groupedTableOptions"
 							@update:query="tableOptions.searchText = $event"
diff --git a/frontend/src2/query/components/QueryBuilderToolbar.vue b/frontend/src2/query/components/QueryBuilderToolbar.vue
index c2dafe10..385ff76e 100644
--- a/frontend/src2/query/components/QueryBuilderToolbar.vue
+++ b/frontend/src2/query/components/QueryBuilderToolbar.vue
@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { CodeIcon, PlayIcon, Scroll } from 'lucide-vue-next'
+import { PlayIcon, Scroll } from 'lucide-vue-next'
 import { inject, ref } from 'vue'
 import { Query } from '../query'
 import ViewSQLDialog from './ViewSQLDialog.vue'
@@ -28,13 +28,15 @@ const actions = [
 
 <template>
 	<div class="flex w-full flex-shrink-0 items-center justify-between bg-white">
-		<div
-			v-show="query.result.executedSQL"
-			class="tnum flex items-center gap-2 text-sm text-gray-600"
-		>
-			<div class="h-2 w-2 rounded-full bg-green-500"></div>
-			<span v-if="query.result.timeTaken == -1"> Fetched from cache </span>
-			<span v-else> Fetched in {{ query.result.timeTaken }} ms </span>
+		<div>
+			<div
+				v-show="query.result.executedSQL"
+				class="tnum flex items-center gap-2 text-sm text-gray-600"
+			>
+				<div class="h-2 w-2 rounded-full bg-green-500"></div>
+				<span v-if="query.result.timeTaken == -1"> Fetched from cache </span>
+				<span v-else> Fetched in {{ query.result.timeTaken }} ms </span>
+			</div>
 		</div>
 		<div class="flex items-center gap-2">
 			<template v-for="(action, idx) in actions" :key="idx">
diff --git a/frontend/src2/query/query.ts b/frontend/src2/query/query.ts
index e3722212..e5d53d8f 100644
--- a/frontend/src2/query/query.ts
+++ b/frontend/src2/query/query.ts
@@ -21,7 +21,6 @@ import {
 	QueryResult,
 	Rename,
 	SelectArgs,
-	Source,
 	SourceArgs,
 	SummarizeArgs,
 	UnionArgs
@@ -39,7 +38,6 @@ import {
 	mutate,
 	order_by,
 	pivot_wider,
-	query_table,
 	remove,
 	rename,
 	select,
@@ -68,7 +66,7 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 
 		activeOperationIdx: -1,
 		activeEditIndex: -1,
-		source: computed(() => ({} as Source)),
+		source: computed(() => ''),
 		currentOperations: computed(() => [] as Operation[]),
 		activeEditOperation: computed(() => ({} as Operation)),
 
@@ -168,11 +166,22 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 
 	// @ts-ignore
 	query.source = computed(() => {
-		const sourceOp = query.doc.operations.find((op) => op.type === 'source')
-		if (!sourceOp) return {} as Source
-		return sourceOp as Source
+		const operations = query.getOperationsForExecution()
+		return getDataSource(operations)
 	})
 
+	function getDataSource(operations: Operation[]): string {
+		const source = operations.find((op) => op.type === 'source')
+		if (!source) return ''
+		if (source.table.type === 'table') {
+			return source.table.data_source
+		}
+		if (source.table.type === 'query' && 'query' in source.table) {
+			return getDataSource(source.table.operations!)
+		}
+		return ''
+	}
+
 	// @ts-ignore
 	query.currentOperations = computed(() => {
 		const operations = [...query.doc.operations]
@@ -218,7 +227,6 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 			op.table.operations = queryTable.getOperationsForExecution()
 		}
 
-
 		if (query.dashboardFilters.filters?.length) {
 			_operations.push(filter_group(query.dashboardFilters))
 			query.dashboardFilters = {} as FilterGroupArgs
diff --git a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
index a9946df9..cbc08ca8 100644
--- a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
+++ b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
@@ -111,9 +111,6 @@ def get_right_table(self, join_args):
         for col in join_args.select_columns:
             select_columns.add(col.column_name)
 
-        if join_args.right_column:
-            select_columns.add(join_args.right_column.column_name)
-
         if join_args.join_condition and join_args.join_condition.right_column:
             select_columns.add(join_args.join_condition.right_column.column_name)
 
@@ -164,9 +161,10 @@ def left_eq_right_condition(left_column, right_column):
 
             frappe.throw("Join condition is not valid")
 
-        if join_args.join_condition and join_args.join_condition.join_expression:
+        join_condition = join_args.join_condition
+        if join_condition.join_expression and join_condition.join_expression.expression:
             return self.evaluate_expression(
-                join_args.join_condition.join_expression.expression,
+                join_condition.join_expression.expression,
                 {
                     "t1": _,
                     "t2": right_table,
@@ -174,8 +172,8 @@ def left_eq_right_condition(left_column, right_column):
             )
         else:
             return left_eq_right_condition(
-                join_args.left_column or join_args.join_condition.left_column,
-                join_args.right_column or join_args.join_condition.right_column,
+                join_condition.left_column,
+                join_condition.right_column,
             )
 
     def rename_duplicate_columns(self, right_table):
@@ -183,7 +181,7 @@ def rename_duplicate_columns(self, right_table):
         query_columns = set(query.columns)
         right_table_columns = set(right_table.columns)
         right_table_name = get_ibis_table_name(right_table)
-        right_table_name = frappe.scrub(right_table_name)
+        right_table_name = sanitize_name(right_table_name)
 
         duplicate_columns = query_columns.intersection(right_table_columns)
         if not duplicate_columns:
@@ -193,7 +191,7 @@ def is_conflicting(col):
             return col in query_columns or col in right_table_columns
 
         def get_new_name(col):
-            new_name = f"{right_table_name}.{col}"
+            new_name = f"{right_table_name}_{col}"
             if not is_conflicting(new_name):
                 return new_name
 
@@ -301,7 +299,7 @@ def apply_select(self, select_args):
 
     def apply_rename(self, rename_args):
         old_name = rename_args.column.column_name
-        new_name = frappe.scrub(rename_args.new_name)
+        new_name = sanitize_name(rename_args.new_name)
         return self.query.rename(**{new_name: old_name})
 
     def apply_remove(self, remove_args):
@@ -324,7 +322,7 @@ def get_ibis_dtype(self, data_type):
         }[data_type]
 
     def apply_mutate(self, mutate_args):
-        new_name = frappe.scrub(mutate_args.new_name)
+        new_name = sanitize_name(mutate_args.new_name)
         dtype = self.get_ibis_dtype(mutate_args.data_type)
         new_column = self.evaluate_expression(mutate_args.expression.expression)
         new_column = new_column.cast(dtype)
@@ -332,7 +330,7 @@ def apply_mutate(self, mutate_args):
 
     def apply_summary(self, summarize_args):
         aggregates = {
-            frappe.scrub(measure.measure_name): self.translate_measure(measure)
+            sanitize_name(measure.measure_name): self.translate_measure(measure)
             for measure in summarize_args.measures
         }
         group_bys = [
@@ -361,7 +359,7 @@ def apply_pivot(self, pivot_args, pivot_type):
             for dimension in pivot_args["columns"]
         }
         values = {
-            frappe.scrub(measure.measure_name): self.translate_measure(measure)
+            sanitize_name(measure.measure_name): self.translate_measure(measure)
             for measure in pivot_args["values"]
         }
 
@@ -416,7 +414,7 @@ def translate_measure(self, measure):
         if "expression" in measure:
             column = self.evaluate_expression(measure.expression.expression)
             dtype = self.get_ibis_dtype(measure.data_type)
-            measure_name = frappe.scrub(measure.measure_name)
+            measure_name = sanitize_name(measure.measure_name)
             return column.cast(dtype).name(measure_name)
 
         column = getattr(_, measure.column_name)
@@ -609,3 +607,13 @@ def get_ibis_table_name(table: IbisQuery):
     if not dt:
         return None
     return dt[0].name
+
+
+def sanitize_name(name):
+    return (
+        name.replace(" ", "_")
+        .replace("-", "_")
+        .replace(".", "_")
+        .replace("/", "_")
+        .lower()
+    )

From 2b27bbf5451592a16a606ac27ac3ebafd013daa8 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Sat, 9 Nov 2024 12:44:47 +0530
Subject: [PATCH 12/15] feat: drag and drop columns to re-arrange

---
 frontend/src2/components/DraggableList.vue    |   9 +-
 .../components/ColumnsSelectorDialog.vue      | 132 ++++++++----------
 .../components/SummarySelectorDialog.vue      |  14 +-
 frontend/src2/query/query.ts                  |   9 ++
 4 files changed, 82 insertions(+), 82 deletions(-)

diff --git a/frontend/src2/components/DraggableList.vue b/frontend/src2/components/DraggableList.vue
index a5c4ff5f..a197cba0 100644
--- a/frontend/src2/components/DraggableList.vue
+++ b/frontend/src2/components/DraggableList.vue
@@ -36,7 +36,7 @@ function onChange(e) {
 		@change="onChange"
 	>
 		<template #item="{ element: item, index: idx }">
-			<div class="mb-2 flex items-center gap-1 last:mb-0">
+			<div class="mb-1.5 flex items-center gap-1 last:mb-0">
 				<GripVertical
 					v-if="props.showHandle"
 					class="h-4 w-4 flex-shrink-0 cursor-grab text-gray-500"
@@ -44,15 +44,18 @@ function onChange(e) {
 				<div class="flex flex-1 flex-col justify-center overflow-hidden">
 					<slot name="item" :item="item" :index="idx">
 						<div
-							class="group form-input flex h-7 flex-1 cursor-pointer items-center justify-between px-2"
+							class="group flex flex-1 cursor-pointer items-center justify-between rounded bg-gray-50 py-1.5 px-2 hover:bg-gray-100"
 						>
 							<div class="flex items-center space-x-2">
-								<div>{{ typeof item === 'object' ? item[itemKey] : item }}</div>
+								<slot name="item-content" :item="item" :index="idx">
+									<div>{{ typeof item === 'object' ? item[itemKey] : item }}</div>
+								</slot>
 							</div>
 							<div class="flex items-center space-x-2">
 								<X
 									@click.prevent.stop="items.splice(idx, 1)"
 									class="invisible h-4 w-4 text-gray-600 transition-all hover:text-gray-800 group-hover:visible"
+									stroke-width="1.5"
 								/>
 							</div>
 						</div>
diff --git a/frontend/src2/query/components/ColumnsSelectorDialog.vue b/frontend/src2/query/components/ColumnsSelectorDialog.vue
index 53843884..c12439d9 100644
--- a/frontend/src2/query/components/ColumnsSelectorDialog.vue
+++ b/frontend/src2/query/components/ColumnsSelectorDialog.vue
@@ -1,6 +1,7 @@
 <script setup lang="ts">
-import { CheckSquare, SearchIcon, SquareIcon } from 'lucide-vue-next'
+import { SearchIcon } from 'lucide-vue-next'
 import { computed, inject, ref } from 'vue'
+import DraggableList from '../../components/DraggableList.vue'
 import { QueryResultColumn, SelectArgs } from '../../types/query.types'
 import { Query } from '../query'
 import DataTypeIcon from './DataTypeIcon.vue'
@@ -12,47 +13,42 @@ const emit = defineEmits({
 const showDialog = defineModel()
 
 const query = inject('query') as Query
-const columns = ref<QueryResultColumn[]>([])
-query.getColumnsForSelection().then((cols: QueryResultColumn[]) => {
-	columns.value = cols
-	selectedColumns.value = props.columns
-		? columns.value.filter((c) => props.columns?.column_names.includes(c.name))
-		: []
-})
+const selectedColumns = ref<QueryResultColumn[]>([...query.result.columns])
 
-const columnSearchQuery = ref('')
-const filteredColumns = computed(() => {
-	if (!columns.value.length) return []
-	if (!columnSearchQuery.value) return columns.value
-	return columns.value.filter((column) => {
-		return column.name.toLowerCase().includes(columnSearchQuery.value.toLowerCase())
-	})
+const columnOptions = ref<ColumnOption[]>([])
+query.getColumnsForSelection().then((cols: ColumnOption[]) => {
+	columnOptions.value = cols
 })
 
-const selectedColumns = ref<QueryResultColumn[]>([])
-function isSelected(column: QueryResultColumn) {
-	return selectedColumns.value.includes(column)
-}
-function toggleColumn(column: QueryResultColumn) {
-	const index = selectedColumns.value.indexOf(column)
-	if (index === -1) {
-		selectedColumns.value.push(column)
-	} else {
-		selectedColumns.value.splice(index, 1)
+const columns = ref<HTMLElement | null>(null)
+function addColumn(column: ColumnOption) {
+	if (!column?.value) return
+	if (!selectedColumns.value.find((c) => c.name === column.value)) {
+		selectedColumns.value.push({
+			name: column.value,
+			type: column.data_type,
+		})
+		setTimeout(() => {
+			columns.value?.scrollTo({
+				top: columns.value.scrollHeight,
+				behavior: 'smooth',
+			})
+		}, 100)
 	}
 }
-function toggleSelectAll() {
-	if (areAllSelected.value) {
-		selectedColumns.value = []
-	} else {
-		selectedColumns.value = [...columns.value]
-	}
-}
-const areAllSelected = computed(() => selectedColumns.value.length === columns.value.length)
-const areNoneSelected = computed(() => selectedColumns.value.length === 0)
+
+const confirmDisabled = computed(
+	() =>
+		selectedColumns.value.length === 0 ||
+		selectedColumns.value.map((c) => c.name).join() ===
+			query.result.columns.map((c) => c.name).join()
+)
 function confirmSelection() {
 	emit('select', {
-		column_names: selectedColumns.value.map((c) => c.name),
+		column_names: selectedColumns.value
+			.map((c) => c.name)
+			.filter((c) => c)
+			.filter((c, i, a) => a.indexOf(c) === i),
 	})
 	showDialog.value = false
 }
@@ -68,7 +64,7 @@ function confirmSelection() {
 				{
 					label: 'Confirm',
 					variant: 'solid',
-					disabled: areNoneSelected,
+					disabled: confirmDisabled,
 					onClick: confirmSelection,
 				},
 				{
@@ -79,44 +75,38 @@ function confirmSelection() {
 		}"
 	>
 		<template #body-content>
-			<div class="-mb-5 flex max-h-[20rem] flex-col gap-2 p-0.5">
-				<div class="flex gap-2">
-					<FormControl
-						class="flex-1"
-						autocomplete="off"
-						placeholder="Search by name"
-						v-model="columnSearchQuery"
+			<div class="-mb-7 flex max-h-[22rem] flex-col gap-4 p-0.5 text-base">
+				<Autocomplete
+					:options="
+						columnOptions.filter(
+							(c) => !selectedColumns.find((sc) => sc.name === c.value)
+						)
+					"
+					placeholder="Add column"
+					@update:modelValue="addColumn"
+				>
+					<template #prefix>
+						<SearchIcon class="h-4 w-4 text-gray-500" stroke-width="1.5" />
+					</template>
+				</Autocomplete>
+
+				<div ref="columns" class="relative overflow-y-scroll">
+					<DraggableList
+						v-model:items="selectedColumns"
+						:item-key="'name'"
+						group="columns"
 					>
-						<template #prefix>
-							<SearchIcon class="h-4 w-4 text-gray-500" />
-						</template>
-					</FormControl>
-					<Button @click="toggleSelectAll">
-						{{
-							areAllSelected
-								? `Deselect All (${selectedColumns.length})`
-								: `Select All (${selectedColumns.length})`
-						}}
-					</Button>
-				</div>
-				<div class="flex flex-col overflow-y-scroll rounded border p-0.5 text-base">
-					<template v-for="column in filteredColumns">
-						<div
-							class="flex h-7 flex-shrink-0 cursor-pointer items-center justify-between rounded px-2 hover:bg-gray-100"
-							@click="toggleColumn(column)"
-						>
+						<template #item-content="{ item }">
 							<div class="flex items-center gap-1.5">
-								<DataTypeIcon :columnType="column.type" />
-								<span>{{ column.name }}</span>
+								<DataTypeIcon :columnType="item.type" />
+								<span>{{ item.name }}</span>
 							</div>
-							<component
-								class="h-4 w-4"
-								stroke-width="1.5"
-								:is="isSelected(column) ? CheckSquare : SquareIcon"
-								:class="isSelected(column) ? 'text-gray-900' : 'text-gray-600'"
-							/>
-						</div>
-					</template>
+						</template>
+					</DraggableList>
+
+					<p class="sticky bottom-0 bg-white pt-1.5 text-sm text-gray-500">
+						{{ selectedColumns.length }} columns selected
+					</p>
 				</div>
 			</div>
 		</template>
diff --git a/frontend/src2/query/components/SummarySelectorDialog.vue b/frontend/src2/query/components/SummarySelectorDialog.vue
index 981bb95f..9d8a7386 100644
--- a/frontend/src2/query/components/SummarySelectorDialog.vue
+++ b/frontend/src2/query/components/SummarySelectorDialog.vue
@@ -5,6 +5,7 @@ import { FIELDTYPES } from '../../helpers/constants'
 import {
 	aggregations,
 	ColumnMeasure,
+	ColumnOption,
 	Dimension,
 	DimensionDataType,
 	MeasureDataType,
@@ -20,19 +21,16 @@ const emit = defineEmits({ select: (args: SummarizeArgs) => true })
 const showDialog = defineModel()
 
 const query = inject<Query>('query')!
-const columnOptions = ref<QueryResultColumn[]>([])
-query.getColumnsForSelection().then((cols: QueryResultColumn[]) => {
-	columnOptions.value = cols.map((col) => ({
-		...col,
-		value: col.name,
-	}))
+const columnOptions = ref<ColumnOption[]>([])
+query.getColumnsForSelection().then((cols: ColumnOption[]) => {
+	columnOptions.value = cols
 })
 
 const numberColumns = computed(() =>
-	columnOptions.value.filter((col) => FIELDTYPES.NUMBER.includes(col.type))
+	columnOptions.value.filter((col) => FIELDTYPES.NUMBER.includes(col.data_type))
 )
 const nonNumberColumns = computed(() =>
-	columnOptions.value.filter((col) => !FIELDTYPES.NUMBER.includes(col.type))
+	columnOptions.value.filter((col) => !FIELDTYPES.NUMBER.includes(col.data_type))
 )
 
 const measures = ref<ColumnMeasure[]>(copy((props.summary?.measures as ColumnMeasure[]) || []))
diff --git a/frontend/src2/query/query.ts b/frontend/src2/query/query.ts
index e5d53d8f..0bcbad24 100644
--- a/frontend/src2/query/query.ts
+++ b/frontend/src2/query/query.ts
@@ -19,6 +19,7 @@ import {
 	OrderByArgs,
 	PivotWiderArgs,
 	QueryResult,
+	QueryResultColumn,
 	Rename,
 	SelectArgs,
 	SourceArgs,
@@ -658,6 +659,14 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 		return call(method, {
 			use_live_connection: query.doc.use_live_connection,
 			operations: operationsForExecution,
+		}).then((columns: QueryResultColumn[]) => {
+			return columns.map((column) => ({
+				label: column.name,
+				value: column.name,
+				description: column.type,
+				query: query.doc.name,
+				data_type: column.type,
+			}))
 		})
 	}
 

From d065387a59b6479c1c2dd367195c8238667b98c4 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Sat, 9 Nov 2024 13:20:22 +0530
Subject: [PATCH 13/15] fix: chart legends doesn't display custom measure name

---
 .../src2/charts/components/NumberChart.vue    |  8 ++--
 frontend/src2/charts/helpers.ts               | 39 +++++++++++++------
 frontend/src2/helpers/index.ts                | 11 ++++++
 frontend/src2/query/helpers.ts                |  2 +-
 frontend/src2/query/query.ts                  |  2 +-
 .../insights_data_source_v3/ibis_utils.py     |  2 +
 6 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/frontend/src2/charts/components/NumberChart.vue b/frontend/src2/charts/components/NumberChart.vue
index a84d8b4e..f13feba9 100644
--- a/frontend/src2/charts/components/NumberChart.vue
+++ b/frontend/src2/charts/components/NumberChart.vue
@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { computed } from 'vue'
-import { formatNumber, getShortNumber, scrub } from '../../helpers'
+import { formatNumber, getShortNumber, sanitizeColumnName } from '../../helpers'
 import { NumberChartConfig } from '../../types/chart.types'
 import { QueryResult } from '../../types/query.types'
 import Sparkline from './Sparkline.vue'
@@ -25,7 +25,8 @@ const numberValuesPerColumn = computed(() => {
 	if (!props.result?.rows?.length) return {}
 
 	return numberColumns.value.reduce((acc: any, measure_name: string) => {
-		acc[scrub(measure_name)] = props.result.rows.map((row: any) => row[scrub(measure_name)])
+		const colName = sanitizeColumnName(measure_name)
+		acc[colName] = props.result.rows.map((row: any) => row[colName])
 		return acc
 	}, {})
 })
@@ -36,7 +37,8 @@ const cards = computed(() => {
 	if (!Object.keys(numberValuesPerColumn.value).length) return []
 
 	return numberColumns.value.map((measure_name: string) => {
-		const numberValues = numberValuesPerColumn.value[scrub(measure_name)]
+		const colName = sanitizeColumnName(measure_name)
+		const numberValues = numberValuesPerColumn.value[colName]
 		const currentValue = numberValues[numberValues.length - 1]
 		const previousValue = numberValues[numberValues.length - 2]
 		const delta = config.value.negative_is_better
diff --git a/frontend/src2/charts/helpers.ts b/frontend/src2/charts/helpers.ts
index 29844db3..d795d3da 100644
--- a/frontend/src2/charts/helpers.ts
+++ b/frontend/src2/charts/helpers.ts
@@ -1,5 +1,12 @@
 import { graphic } from 'echarts/core'
-import { copy, ellipsis, formatNumber, getShortNumber, getUniqueId } from '../helpers'
+import {
+	copy,
+	ellipsis,
+	formatNumber,
+	getShortNumber,
+	getUniqueId,
+	sanitizeColumnName,
+} from '../helpers'
 import { FIELDTYPES } from '../helpers/constants'
 import { column, getFormattedDate } from '../query/helpers'
 import useQuery from '../query/query'
@@ -9,6 +16,7 @@ import {
 	ChartConfig,
 	DountChartConfig,
 	LineChartConfig,
+	Series,
 	SeriesLine,
 } from '../types/chart.types'
 import {
@@ -83,15 +91,17 @@ export function getLineChartOptions(config: LineChartConfig, result: QueryResult
 			const serie = getSerie(config, c.name) as SeriesLine
 
 			const is_right_axis = serie.align === 'Right'
+			const type = serie.type?.toLowerCase() || 'line'
 			const smooth = serie.smooth ?? config.y_axis.smooth
 			const show_data_points = serie.show_data_points ?? config.y_axis.show_data_points
 			const show_area = serie.show_area ?? config.y_axis.show_area
 			const show_data_labels = serie.show_data_labels ?? config.y_axis.show_data_labels
 			const color = serie.color?.[0] || colors[idx]
+			const name = serie.measure.measure_name || c.name
 
 			return {
-				type: 'line',
-				name: c.name,
+				type,
+				name,
 				data: getSeriesData(c.name),
 				color: color,
 				yAxisIndex: is_right_axis ? 1 : 0,
@@ -189,17 +199,18 @@ export function getBarChartOptions(config: BarChartConfig, result: QueryResult,
 			const serie = getSerie(config, c.name)
 			const is_right_axis = serie.align === 'Right'
 
-			const stack = config.y_axis.stack
-			const show_data_labels = serie.show_data_labels ?? config.y_axis.show_data_labels
 			const color = serie.color?.[0] || colors[idx]
 			const type = serie.type?.toLowerCase() || 'bar'
+			const stack = type === 'bar' && config.y_axis.stack ? 'stack' : undefined
+			const show_data_labels = serie.show_data_labels ?? config.y_axis.show_data_labels
 
 			const data = getSeriesData(c.name)
+			const name = serie.measure.measure_name || c.name
 
 			return {
 				type,
-				stack: type === 'bar' && stack ? 'stack' : undefined,
-				name: c.name,
+				stack,
+				name,
 				data: swapAxes ? data.reverse() : data,
 				color: color,
 				label: {
@@ -226,19 +237,25 @@ export function getBarChartOptions(config: BarChartConfig, result: QueryResult,
 	}
 }
 
-function getSerie(config: AxisChartConfig, number_column: string) {
+function getSerie(config: AxisChartConfig, number_column: string): Series {
 	let serie
 	if (!config.split_by?.column_name) {
-		serie = config.y_axis.series.find((s) => s.measure.measure_name === number_column)
+		serie = config.y_axis.series.find(
+			(s) => sanitizeColumnName(s.measure.measure_name) === number_column
+		)
 	} else {
 		let seriesCount = config.y_axis.series.filter((s) => s.measure.measure_name).length
 		if (seriesCount === 1) {
 			serie = config.y_axis.series[0]
 		} else {
-			serie = config.y_axis.series.find((s) => number_column.includes(s.measure.measure_name))
+			serie = config.y_axis.series.find((s) =>
+				number_column.includes(sanitizeColumnName(s.measure.measure_name))
+			)
 		}
 	}
-	serie = serie || config.y_axis.series[0]
+	if (!serie) {
+		throw new Error(`Cannot find series for column ${number_column}`)
+	}
 	return serie
 }
 
diff --git a/frontend/src2/helpers/index.ts b/frontend/src2/helpers/index.ts
index e52d3d68..6d81a4f0 100644
--- a/frontend/src2/helpers/index.ts
+++ b/frontend/src2/helpers/index.ts
@@ -270,3 +270,14 @@ export function toOptions(arr: any[], map: Record<OptionKey, string>) {
 		return item
 	})
 }
+
+export function sanitizeColumnName(name: string) {
+	return name
+		.replace(' ', '_')
+		.replace('-', '_')
+		.replace('.', '_')
+		.replace('/', '_')
+		.replace('(', '_')
+		.replace(')', '_')
+		.toLowerCase()
+}
diff --git a/frontend/src2/query/helpers.ts b/frontend/src2/query/helpers.ts
index 2d198e47..ca798538 100644
--- a/frontend/src2/query/helpers.ts
+++ b/frontend/src2/query/helpers.ts
@@ -81,7 +81,7 @@ export const count = (): Measure => ({
 	column_name: 'count',
 	data_type: 'Integer',
 	aggregation: 'count',
-	measure_name: 'count(*)',
+	measure_name: 'count_of_rows',
 })
 export const operator = (operator: FilterOperator): FilterOperator => operator
 export const value = (value: FilterValue): FilterValue => value
diff --git a/frontend/src2/query/query.ts b/frontend/src2/query/query.ts
index 0bcbad24..513cc4c0 100644
--- a/frontend/src2/query/query.ts
+++ b/frontend/src2/query/query.ts
@@ -157,7 +157,7 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 					return {
 						aggregation: 'sum',
 						column_name: column.name,
-						measure_name: `sum(${column.name})`,
+						measure_name: `sum_of_${column.name}`,
 						data_type: column.type as MeasureDataType,
 					}
 				}),
diff --git a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
index cbc08ca8..b0abc456 100644
--- a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
+++ b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py
@@ -615,5 +615,7 @@ def sanitize_name(name):
         .replace("-", "_")
         .replace(".", "_")
         .replace("/", "_")
+        .replace("(", "_")
+        .replace(")", "_")
         .lower()
     )

From 015e2130b5b9f648a0b26eb98ce53b480a59d384 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Sat, 9 Nov 2024 14:43:12 +0530
Subject: [PATCH 14/15] fix: special characters in `db_name`

---
 .../insights_data_source_v3.py                        | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py
index 25b09545..ed015f3b 100644
--- a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py
+++ b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py
@@ -204,10 +204,17 @@ def _get_connection_string(self):
 
         frappe.throw(f"Unsupported database type: {self.database_type}")
 
+    def get_quoted_db_name(self):
+        if not self.database_name:
+            return None
+        quote_start = self._get_ibis_backend().dialect.QUOTE_START
+        quote_end = self._get_ibis_backend().dialect.QUOTE_END
+        return f"{quote_start}{self.database_name}{quote_end}"
+
     def test_connection(self, raise_exception=False):
         try:
             db = self._get_ibis_backend()
-            db.list_tables()
+            db.list_tables(database=self.get_quoted_db_name())
             return True
         except Exception as e:
             frappe.log_error("Testing Data Source connection failed", e)
@@ -218,7 +225,7 @@ def update_table_list(self, force=False):
         blacklist_patterns = ["^_", "^sqlite_"]
         blacklisted = lambda table: any(re.match(p, table) for p in blacklist_patterns)
         remote_db = self._get_ibis_backend()
-        remote_tables = remote_db.list_tables()
+        remote_tables = remote_db.list_tables(database=self.get_quoted_db_name())
         remote_tables = [t for t in remote_tables if not blacklisted(t)]
 
         if not remote_tables:

From c351825abc0dbe36b00bd5881dc6a5434ceb0b31 Mon Sep 17 00:00:00 2001
From: Saqib Ansari <nextchamp.saqib@gmail.com>
Date: Sat, 9 Nov 2024 15:03:53 +0530
Subject: [PATCH 15/15] fix: don't check permissions in v2

---
 insights/api/dashboards.py | 11 +----------
 insights/api/queries.py    |  8 --------
 2 files changed, 1 insertion(+), 18 deletions(-)

diff --git a/insights/api/dashboards.py b/insights/api/dashboards.py
index 6ad2c557..d52cfaf5 100644
--- a/insights/api/dashboards.py
+++ b/insights/api/dashboards.py
@@ -2,17 +2,12 @@
 
 from insights.api.permissions import is_private
 from insights.decorators import insights_whitelist, validate_type
-from insights.insights.doctype.insights_team.insights_team import (
-    get_allowed_resources_for_user,
-    get_permission_filter,
-)
 
 
 @insights_whitelist()
 def get_dashboard_list():
     dashboards = frappe.get_list(
         "Insights Dashboard",
-        filters={**get_permission_filter("Insights Dashboard")},
         fields=["name", "title", "modified", "_liked_by"],
     )
     for dashboard in dashboards:
@@ -54,10 +49,6 @@ def create_dashboard(title):
 
 @insights_whitelist()
 def get_dashboard_options(chart):
-    allowed_dashboards = get_allowed_resources_for_user("Insights Dashboard")
-    if not allowed_dashboards:
-        return []
-
     # find all dashboards that don't have the chart within the allowed dashboards
     Dashboard = frappe.qb.DocType("Insights Dashboard")
     DashboardItem = frappe.qb.DocType("Insights Dashboard Item")
@@ -67,7 +58,7 @@ def get_dashboard_options(chart):
         .left_join(DashboardItem)
         .on(Dashboard.name == DashboardItem.parent)
         .select(Dashboard.name.as_("value"), Dashboard.title.as_("label"))
-        .where(Dashboard.name.isin(allowed_dashboards) & (DashboardItem.chart != chart))
+        .where(DashboardItem.chart != chart)
         .groupby(Dashboard.name)
         .run(as_dict=True)
     )
diff --git a/insights/api/queries.py b/insights/api/queries.py
index 9d2de2ac..f87e0281 100644
--- a/insights/api/queries.py
+++ b/insights/api/queries.py
@@ -1,17 +1,10 @@
 import frappe
 
 from insights.decorators import insights_whitelist
-from insights.insights.doctype.insights_team.insights_team import (
-    get_allowed_resources_for_user,
-)
 
 
 @insights_whitelist()
 def get_queries():
-    allowed_queries = get_allowed_resources_for_user("Insights Query")
-    if not allowed_queries:
-        return []
-
     Query = frappe.qb.DocType("Insights Query")
     QueryChart = frappe.qb.DocType("Insights Chart")
     DataSource = frappe.qb.DocType("Insights Data Source")
@@ -39,7 +32,6 @@ def get_queries():
             QueryChart.chart_type,
             DataSource.title.as_("data_source_title"),
         )
-        .where(Query.name.isin(allowed_queries))
         .groupby(
             Query.name,
             User.full_name.as_("owner_name"),