From 79f3d1e1c5c4f3d96779673d5ce2433ccf478d23 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 29 Feb 2024 01:42:27 +1100 Subject: [PATCH 1/6] Wrapper --- .../controllers/connectionDisplayData.node.ts | 6 ++-- src/platform/interpreter/helpers.ts | 35 +++++++++++++++---- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/notebooks/controllers/connectionDisplayData.node.ts b/src/notebooks/controllers/connectionDisplayData.node.ts index 3cff4f2567c..2220fac3e89 100644 --- a/src/notebooks/controllers/connectionDisplayData.node.ts +++ b/src/notebooks/controllers/connectionDisplayData.node.ts @@ -22,7 +22,7 @@ import { import { DataScience } from '../../platform/common/utils/localize'; import { getJupyterDisplayName } from '../../kernels/jupyter/connection/jupyterServerProviderRegistry'; import { - getCachedEnvironments, + getCachedEnvironment, isCondaEnvironmentWithoutPython, resolvedPythonEnvToJupyterEnv } from '../../platform/interpreter/helpers'; @@ -68,9 +68,7 @@ export class ConnectionDisplayDataProvider implements IConnectionDisplayDataProv ) { const updateInterpreterInfo = (e: PythonEnvironment[]) => { const changedEnv = e.find((env) => env.id === connection.interpreter?.id); - const interpreter = resolvedPythonEnvToJupyterEnv( - getCachedEnvironments().find((env) => env.id === changedEnv?.id) - ); + const interpreter = resolvedPythonEnvToJupyterEnv(getCachedEnvironment(changedEnv)); if (connection.kind === 'startUsingPythonInterpreter' && interpreter) { connection.updateInterpreter(interpreter); const newLabel = getDisplayNameOrNameOfKernelConnection(connection); diff --git a/src/platform/interpreter/helpers.ts b/src/platform/interpreter/helpers.ts index 9122f28e86a..f516f868c03 100644 --- a/src/platform/interpreter/helpers.ts +++ b/src/platform/interpreter/helpers.ts @@ -123,9 +123,23 @@ export async function getInterpreterInfo(interpreter?: { id: string }) { return api.environments.resolveEnvironment(interpreter.id); } +let cachedKnown: typeof pythonApi.environments.known | undefined; +let cachedKnownDict = new Map(); let pythonApi: PythonExtension; export function setPythonApi(api: PythonExtension) { + if (pythonApi === api) { + return; + } pythonApi = api; + api.environments.known.forEach((e) => cachedKnownDict.set(e.id, e)); + api.environments.onDidChangeEnvironments((e) => { + cachedKnown = undefined; + if (e.type === 'remove') { + cachedKnownDict.delete(e.env.id); + } else { + cachedKnownDict.set(e.env.id, e.env); + } + }); } export function isCondaEnvironmentWithoutPython(interpreter?: { id: string }) { @@ -140,6 +154,14 @@ export function isCondaEnvironmentWithoutPython(interpreter?: { id: string }) { return env && getEnvironmentType(env) === EnvironmentType.Conda && !env.executable.uri; } +function getCachedEnvs() { + if (!cachedKnown) { + cachedKnown = pythonApi.environments.known; + // cachedKnownDict.clear(); + // cachedKnown.forEach((i) => cachedKnownDict.set(i.id, i)); + } + return { known: cachedKnown, map: cachedKnownDict }; +} export function getCachedEnvironment(interpreter?: { id: string }) { if (!interpreter) { return; @@ -147,7 +169,7 @@ export function getCachedEnvironment(interpreter?: { id: string }) { if (!pythonApi) { throw new Error('Python API not initialized'); } - return pythonApi.environments.known.find((i) => i.id === interpreter.id); + return cachedKnownDict.get(interpreter.id); } export async function getSysPrefix(interpreter?: { id: string }) { @@ -155,7 +177,7 @@ export async function getSysPrefix(interpreter?: { id: string }) { return; } if (pythonApi) { - const cachedInfo = pythonApi.environments.known.find((i) => i.id === interpreter.id); + const cachedInfo = cachedKnownDict.get(interpreter.id); if (cachedInfo?.executable?.sysPrefix) { return cachedInfo.executable.sysPrefix; } @@ -176,7 +198,7 @@ export function getCachedSysPrefix(interpreter?: { id: string }) { if (!pythonApi) { throw new Error('Python API not initialized'); } - const cachedInfo = pythonApi.environments.known.find((i) => i.id === interpreter.id); + const cachedInfo = cachedKnownDict.get(interpreter.id); return cachedInfo?.executable?.sysPrefix; } export async function getVersion(interpreter?: { id?: string }, ignoreCache = false) { @@ -184,7 +206,7 @@ export async function getVersion(interpreter?: { id?: string }, ignoreCache = fa return; } if (pythonApi && !ignoreCache) { - const cachedInfo = pythonApi.environments.known.find((i) => i.id === interpreter.id); + const cachedInfo = cachedKnownDict.get(interpreter.id); if (cachedInfo?.version) { return cachedInfo.version; } @@ -205,7 +227,7 @@ export function getCachedVersion(interpreter?: { id?: string }) { if (!pythonApi) { throw new Error('Python API not initialized'); } - const cachedInfo = pythonApi.environments.known.find((i) => i.id === interpreter.id); + const cachedInfo = cachedKnownDict.get(interpreter.id); return cachedInfo?.version; } @@ -213,7 +235,8 @@ export function getCachedEnvironments() { if (!pythonApi) { return []; } - return pythonApi.environments.known; + return Array.from(cachedKnownDict.values()); + // return getCachedEnvs().known; } export function resolvedPythonEnvToJupyterEnv(env?: Environment): PythonEnvironment | undefined { if (!env) { From 6718926502f3b466db7bb331d850851f90f445d2 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 29 Feb 2024 01:45:06 +1100 Subject: [PATCH 2/6] Map --- src/platform/interpreter/helpers.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/platform/interpreter/helpers.ts b/src/platform/interpreter/helpers.ts index f516f868c03..74c33a1babe 100644 --- a/src/platform/interpreter/helpers.ts +++ b/src/platform/interpreter/helpers.ts @@ -154,14 +154,14 @@ export function isCondaEnvironmentWithoutPython(interpreter?: { id: string }) { return env && getEnvironmentType(env) === EnvironmentType.Conda && !env.executable.uri; } -function getCachedEnvs() { - if (!cachedKnown) { - cachedKnown = pythonApi.environments.known; - // cachedKnownDict.clear(); - // cachedKnown.forEach((i) => cachedKnownDict.set(i.id, i)); - } - return { known: cachedKnown, map: cachedKnownDict }; -} +// function getCachedEnvs() { +// if (!cachedKnown) { +// cachedKnown = pythonApi.environments.known; +// // cachedKnownDict.clear(); +// // cachedKnown.forEach((i) => cachedKnownDict.set(i.id, i)); +// } +// return { known: cachedKnown, map: cachedKnownDict }; +// } export function getCachedEnvironment(interpreter?: { id: string }) { if (!interpreter) { return; From eb48184e0d0dec3add22ef821da030f7f2de8db1 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 29 Feb 2024 01:49:57 +1100 Subject: [PATCH 3/6] oops --- src/platform/interpreter/helpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/interpreter/helpers.ts b/src/platform/interpreter/helpers.ts index 74c33a1babe..03b36d3c209 100644 --- a/src/platform/interpreter/helpers.ts +++ b/src/platform/interpreter/helpers.ts @@ -123,7 +123,7 @@ export async function getInterpreterInfo(interpreter?: { id: string }) { return api.environments.resolveEnvironment(interpreter.id); } -let cachedKnown: typeof pythonApi.environments.known | undefined; +// let cachedKnown: typeof pythonApi.environments.known | undefined; let cachedKnownDict = new Map(); let pythonApi: PythonExtension; export function setPythonApi(api: PythonExtension) { @@ -133,7 +133,7 @@ export function setPythonApi(api: PythonExtension) { pythonApi = api; api.environments.known.forEach((e) => cachedKnownDict.set(e.id, e)); api.environments.onDidChangeEnvironments((e) => { - cachedKnown = undefined; + // cachedKnown = undefined; if (e.type === 'remove') { cachedKnownDict.delete(e.env.id); } else { From 7e1cd9a551527d2449cd511e806c69213608113d Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 29 Feb 2024 11:15:12 +1100 Subject: [PATCH 4/6] faster yet again --- .../kernelSource/localPythonEnvKernelSourceSelector.node.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/notebooks/controllers/kernelSource/localPythonEnvKernelSourceSelector.node.ts b/src/notebooks/controllers/kernelSource/localPythonEnvKernelSourceSelector.node.ts index 3c89de30fe5..16d44763783 100644 --- a/src/notebooks/controllers/kernelSource/localPythonEnvKernelSourceSelector.node.ts +++ b/src/notebooks/controllers/kernelSource/localPythonEnvKernelSourceSelector.node.ts @@ -181,7 +181,10 @@ export class LocalPythonEnvNotebookKernelSourceSelector .catch(noop); } private getKernelSpecsDir() { - return this.tempDirForKernelSpecs || this.jupyterPaths.getKernelSpecTempRegistrationFolder(); + if (!this.tempDirForKernelSpecs) { + this.tempDirForKernelSpecs = this.jupyterPaths.getKernelSpecTempRegistrationFolder(); + } + return this.tempDirForKernelSpecs; } private apiHooked = false; private async hookupPythonApi() { From 1359e627c5dd77e4165bc311d839a607d5482edd Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 29 Feb 2024 11:21:40 +1100 Subject: [PATCH 5/6] Throttle updates to the kernel cache --- .../finder/localKernelSpecFinderBase.node.ts | 21 ++++++++++++------- .../localKnownPathKernelSpecFinder.node.ts | 2 +- ...ndRelatedNonPythonKernelSpecFinder.node.ts | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts b/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts index 33a7160f29e..b9aa3e09deb 100644 --- a/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts +++ b/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts @@ -280,15 +280,20 @@ export abstract class LocalKernelSpecFinderBase< this.promiseMonitor.push(promise); return promise; } - + private timeouts = new Map(); protected async writeToMementoCache(values: T[], cacheKey: string) { - await this.memento.update( - cacheKey, - JSON.stringify({ - kernels: values.map((item) => item.toJSON()), - extensionVersion: this.env.extensionVersion - }) - ); + this.timeouts.get(cacheKey)?.dispose(); + // This can get called very quickly and very often. + const timer = setTimeout(() => { + void this.memento.update( + cacheKey, + JSON.stringify({ + kernels: values.map((item) => item.toJSON()), + extensionVersion: this.env.extensionVersion + }) + ); + }, 500); + this.timeouts.set(cacheKey, { dispose: () => clearTimeout(timer) }); } protected async isValidCachedKernel(kernel: LocalKernelConnectionMetadata): Promise { switch (kernel.kind) { diff --git a/src/kernels/raw/finder/localKnownPathKernelSpecFinder.node.ts b/src/kernels/raw/finder/localKnownPathKernelSpecFinder.node.ts index 4ad34e5eb08..8e402ec9c3b 100644 --- a/src/kernels/raw/finder/localKnownPathKernelSpecFinder.node.ts +++ b/src/kernels/raw/finder/localKnownPathKernelSpecFinder.node.ts @@ -75,7 +75,7 @@ export class LocalKnownPathKernelSpecFinder } @debounce(100) private writeKernelsToMemento() { - this.writeToMementoCache(Array.from(this._kernels.values()), localKernelSpecsCacheKey()).catch(noop); + this.writeToMementoCache(Array.from(this._kernels.values()), localKernelSpecsCacheKey()); } private async listKernelSpecs(cancelToken: CancellationToken): Promise { const fn = async () => { diff --git a/src/kernels/raw/finder/localPythonAndRelatedNonPythonKernelSpecFinder.node.ts b/src/kernels/raw/finder/localPythonAndRelatedNonPythonKernelSpecFinder.node.ts index 2f7cc416d62..19c2ec75d48 100644 --- a/src/kernels/raw/finder/localPythonAndRelatedNonPythonKernelSpecFinder.node.ts +++ b/src/kernels/raw/finder/localPythonAndRelatedNonPythonKernelSpecFinder.node.ts @@ -286,7 +286,7 @@ export class LocalPythonAndRelatedNonPythonKernelSpecFinder extends LocalKernelS this._onDidChangeKernels.fire(); const kernels = Array.from(this._kernels.values()); this.updateCachePromise = this.updateCachePromise.finally(() => - this.writeToMementoCache(kernels, localPythonKernelsCacheKey()).catch(noop) + this.writeToMementoCache(kernels, localPythonKernelsCacheKey()) ); return this.updateCachePromise; } From 6023d66a6fad113f2913cb20fd11e02f3b0beeb0 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 29 Feb 2024 11:31:10 +1100 Subject: [PATCH 6/6] oops --- src/kernels/raw/finder/localKernelSpecFinderBase.node.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts b/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts index b9aa3e09deb..d5d1812c74f 100644 --- a/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts +++ b/src/kernels/raw/finder/localKernelSpecFinderBase.node.ts @@ -281,7 +281,7 @@ export abstract class LocalKernelSpecFinderBase< return promise; } private timeouts = new Map(); - protected async writeToMementoCache(values: T[], cacheKey: string) { + protected writeToMementoCache(values: T[], cacheKey: string) { this.timeouts.get(cacheKey)?.dispose(); // This can get called very quickly and very often. const timer = setTimeout(() => {