Skip to content

Commit

Permalink
Merge branch 'develop' into wip/sb/batch-invalidations
Browse files Browse the repository at this point in the history
  • Loading branch information
somebody1234 committed Jan 8, 2025
2 parents 6c3a972 + b72eb92 commit 7cd1853
Show file tree
Hide file tree
Showing 49 changed files with 5,226 additions and 381 deletions.
9 changes: 9 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -504,10 +504,13 @@ jobs:
ENSO_CLOUD_ENVIRONMENT: ${{ vars.ENSO_CLOUD_ENVIRONMENT }}
ENSO_CLOUD_GOOGLE_ANALYTICS_TAG: ${{ vars.ENSO_CLOUD_GOOGLE_ANALYTICS_TAG }}
ENSO_CLOUD_SENTRY_DSN: ${{ vars.ENSO_CLOUD_SENTRY_DSN }}
ENSO_CLOUD_SENTRY_ORGANIZATION: ${{ vars.ENSO_CLOUD_SENTRY_ORGANIZATION }}
ENSO_CLOUD_SENTRY_PROJECT: ${{ vars.ENSO_CLOUD_SENTRY_PROJECT }}
ENSO_CLOUD_STRIPE_KEY: ${{ vars.ENSO_CLOUD_STRIPE_KEY }}
ENSO_IDE_AG_GRID_LICENSE_KEY: ${{ vars.ENSO_AG_GRID_LICENSE_KEY }}
ENSO_IDE_MAPBOX_API_TOKEN: ${{ vars.ENSO_MAPBOX_API_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
- if: failure() && runner.os == 'Windows'
name: List files if failed (Windows)
run: Get-ChildItem -Force -Recurse
Expand Down Expand Up @@ -573,10 +576,13 @@ jobs:
ENSO_CLOUD_ENVIRONMENT: ${{ vars.ENSO_CLOUD_ENVIRONMENT }}
ENSO_CLOUD_GOOGLE_ANALYTICS_TAG: ${{ vars.ENSO_CLOUD_GOOGLE_ANALYTICS_TAG }}
ENSO_CLOUD_SENTRY_DSN: ${{ vars.ENSO_CLOUD_SENTRY_DSN }}
ENSO_CLOUD_SENTRY_ORGANIZATION: ${{ vars.ENSO_CLOUD_SENTRY_ORGANIZATION }}
ENSO_CLOUD_SENTRY_PROJECT: ${{ vars.ENSO_CLOUD_SENTRY_PROJECT }}
ENSO_CLOUD_STRIPE_KEY: ${{ vars.ENSO_CLOUD_STRIPE_KEY }}
ENSO_IDE_AG_GRID_LICENSE_KEY: ${{ vars.ENSO_AG_GRID_LICENSE_KEY }}
ENSO_IDE_MAPBOX_API_TOKEN: ${{ vars.ENSO_MAPBOX_API_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
- if: failure() && runner.os == 'Windows'
name: List files if failed (Windows)
run: Get-ChildItem -Force -Recurse
Expand Down Expand Up @@ -640,10 +646,13 @@ jobs:
ENSO_CLOUD_ENVIRONMENT: ${{ vars.ENSO_CLOUD_ENVIRONMENT }}
ENSO_CLOUD_GOOGLE_ANALYTICS_TAG: ${{ vars.ENSO_CLOUD_GOOGLE_ANALYTICS_TAG }}
ENSO_CLOUD_SENTRY_DSN: ${{ vars.ENSO_CLOUD_SENTRY_DSN }}
ENSO_CLOUD_SENTRY_ORGANIZATION: ${{ vars.ENSO_CLOUD_SENTRY_ORGANIZATION }}
ENSO_CLOUD_SENTRY_PROJECT: ${{ vars.ENSO_CLOUD_SENTRY_PROJECT }}
ENSO_CLOUD_STRIPE_KEY: ${{ vars.ENSO_CLOUD_STRIPE_KEY }}
ENSO_IDE_AG_GRID_LICENSE_KEY: ${{ vars.ENSO_AG_GRID_LICENSE_KEY }}
ENSO_IDE_MAPBOX_API_TOKEN: ${{ vars.ENSO_MAPBOX_API_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
- if: failure() && runner.os == 'Windows'
name: List files if failed (Windows)
run: Get-ChildItem -Force -Recurse
Expand Down
27 changes: 20 additions & 7 deletions app/gui/src/dashboard/hooks/backendUploadFilesHooks.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,11 @@ import type { MergeValuesOfObjectUnion } from 'enso-common/src/utilities/data/ob
import { useId, useState } from 'react'
import { toast } from 'react-toastify'

// The number of bytes in 1 megabyte.
/** The number of bytes in 1 megabyte. */
const MB_BYTES = 1_000_000
const S3_CHUNK_SIZE_MB = Math.round(S3_CHUNK_SIZE_BYTES / MB_BYTES)
/** The maximum number of file chunks to upload at the same time. */
const FILE_UPLOAD_CONCURRENCY = 5

/** A function to upload files. */
export function useUploadFiles(backend: Backend, category: Category) {
Expand Down Expand Up @@ -409,20 +411,31 @@ export function useUploadFileMutation(
body,
file,
])
let i = 0
let completedChunkCount = 0
const parts: S3MultipartPart[] = []
for (const [url, i] of Array.from(
presignedUrls,
(presignedUrl, index) => [presignedUrl, index] as const,
)) {
parts.push(await uploadFileChunkMutation.mutateAsync([url, file, i]))
const newSentMb = Math.min((i + 1) * S3_CHUNK_SIZE_MB, fileSizeMb)
const uploadNextChunk = async (): Promise<void> => {
const currentI = i
const url = presignedUrls[i]
if (url == null) {
return
}
i += 1
const promise = uploadFileChunkMutation.mutateAsync([url, file, currentI])
// Queue the next chunk to be uploaded after this one.
const fullPromise = promise.then(uploadNextChunk)
parts[currentI] = await promise
completedChunkCount += 1
const newSentMb = Math.min(completedChunkCount * S3_CHUNK_SIZE_MB, fileSizeMb)
setSentMb(newSentMb)
options.onChunkSuccess?.({
event: 'chunk',
sentMb: newSentMb,
totalMb: fileSizeMb,
})
return fullPromise
}
await Promise.all(Array.from({ length: FILE_UPLOAD_CONCURRENCY }).map(uploadNextChunk))
const result = await uploadFileEndMutation.mutateAsync([
{
parentDirectoryId: body.parentDirectoryId,
Expand Down
4 changes: 2 additions & 2 deletions app/ide-desktop/client/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ export default defineConfig({
testDir: './tests',
forbidOnly: !!process.env.CI,
workers: 1,
timeout: 120000,
timeout: 180000,
reportSlowTests: { max: 5, threshold: 60000 },
globalSetup: './tests/setup.ts',
expect: {
timeout: 5000,
timeout: 30000,
toHaveScreenshot: { threshold: 0 },
},
use: {
Expand Down
21 changes: 14 additions & 7 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -2283,12 +2283,13 @@ lazy val `language-server` = (project in file("engine/language-server"))
javaModuleName := "org.enso.language.server",
Compile / moduleDependencies ++=
Seq(
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion,
"org.slf4j" % "slf4j-api" % slf4jVersion,
"commons-cli" % "commons-cli" % commonsCliVersion,
"commons-io" % "commons-io" % commonsIoVersion,
"com.google.flatbuffers" % "flatbuffers-java" % flatbuffersVersion,
"org.eclipse.jgit" % "org.eclipse.jgit" % jgitVersion
"org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion,
"org.slf4j" % "slf4j-api" % slf4jVersion,
"commons-cli" % "commons-cli" % commonsCliVersion,
"commons-io" % "commons-io" % commonsIoVersion,
"com.google.flatbuffers" % "flatbuffers-java" % flatbuffersVersion,
"org.eclipse.jgit" % "org.eclipse.jgit" % jgitVersion,
"org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion
),
Compile / internalModuleDependencies := Seq(
(`akka-wrapper` / Compile / exportedModule).value,
Expand Down Expand Up @@ -3614,13 +3615,17 @@ lazy val `engine-runner` = project
val epbLang =
(`runtime-language-epb` / Compile / fullClasspath).value
.map(_.data.getAbsolutePath)
val langServer =
(`language-server` / Compile / fullClasspath).value
.map(_.data.getAbsolutePath)
val core = (
runnerDeps ++
runtimeDeps ++
loggingDeps ++
replDebugInstr ++
runtimeServerInstr ++
idExecInstr ++
langServer ++
epbLang
).distinct
val stdLibsJars =
Expand Down Expand Up @@ -3722,6 +3727,7 @@ lazy val `engine-runner` = project
"org.jline",
"io.methvin.watchservice",
"zio.internal",
"zio",
"org.enso.runner",
"sun.awt",
"sun.java2d",
Expand All @@ -3733,7 +3739,8 @@ lazy val `engine-runner` = project
"akka.http",
"org.enso.base",
"org.enso.image",
"org.enso.table"
"org.enso.table",
"org.eclipse.jgit"
)
)
}
Expand Down
14 changes: 5 additions & 9 deletions build/build/src/ci_gen/job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,11 @@ pub fn expose_gui_vars(step: Step) -> Step {
}

/// Expose variables for debugging purposes.
pub fn expose_debugging_vars(os: OS, step: Step) -> Step {
pub fn expose_debugging_vars(step: Step) -> Step {
use crate::ide::web::env::*;
match os {
OS::Windows => step
.with_secret_exposed(secret::SENTRY_AUTH_TOKEN)
.with_variable_exposed(ENSO_CLOUD_SENTRY_ORGANIZATION)
.with_variable_exposed(ENSO_CLOUD_SENTRY_PROJECT),
_ => step,
}
step.with_secret_exposed(secret::SENTRY_AUTH_TOKEN)
.with_variable_exposed(ENSO_CLOUD_SENTRY_ORGANIZATION)
.with_variable_exposed(ENSO_CLOUD_SENTRY_PROJECT)
}

#[derive(Clone, Copy, Debug)]
Expand Down Expand Up @@ -582,7 +578,7 @@ pub enum PackagingTarget {
pub fn prepare_packaging_steps(os: OS, step: Step, packaging_target: PackagingTarget) -> Vec<Step> {
let step = expose_gui_vars(step);
let step = if packaging_target == PackagingTarget::Release {
expose_debugging_vars(os, step)
expose_debugging_vars(step)
} else {
step
};
Expand Down
32 changes: 29 additions & 3 deletions engine/common/src/main/java/org/enso/common/ContextFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.enso.logger.JulHandler;
import org.enso.logging.config.LoggerSetup;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Engine;
import org.graalvm.polyglot.HostAccess;
import org.graalvm.polyglot.io.MessageTransport;
import org.slf4j.event.Level;
Expand Down Expand Up @@ -52,6 +53,7 @@ public final class ContextFactory {
private String checkForWarnings;
private int warningsLimit = 100;
private java.util.Map<String, String> options = new HashMap<>();
private String runtimerServerKey;
private boolean enableDebugServer;

private ContextFactory() {}
Expand Down Expand Up @@ -145,6 +147,11 @@ public ContextFactory options(Map<String, String> options) {
return this;
}

public ContextFactory enableRuntimeServerInfoKey(String keyName) {
this.runtimerServerKey = keyName;
return this;
}

public ContextFactory checkForWarnings(String fqnOfMethod) {
this.checkForWarnings = fqnOfMethod;
return this;
Expand All @@ -161,6 +168,16 @@ public Context build() {
}
var julLogLevel = Converter.toJavaLevel(logLevel);
var logLevelName = julLogLevel.getName();
var inAOTMode = java.lang.Boolean.getBoolean("com.oracle.graalvm.isaot");
java.util.Map<String, String> engineOptions = null;
if (runtimerServerKey != null) {
if (!inAOTMode) {
options.put(runtimerServerKey, "true");
} else {
engineOptions = new java.util.HashMap<>();
engineOptions.put(runtimerServerKey, "true");
}
}
var builder =
Context.newBuilder()
.allowExperimentalOptions(true)
Expand Down Expand Up @@ -189,9 +206,6 @@ public Context build() {
if (enableDebugServer) {
builder.option(DebugServerInfo.ENABLE_OPTION, "true");
}
if (messageTransport != null) {
builder.serverTransport(messageTransport);
}
builder.option(RuntimeOptions.LOG_LEVEL, logLevelName);
var logHandler = JulHandler.get();
var logLevels = LoggerSetup.get().getConfig().getLoggers();
Expand Down Expand Up @@ -228,6 +242,18 @@ public Context build() {
.allowCreateThread(true);
}

if (inAOTMode) {
// In AOT mode one must not use a shared engine; the latter causes issues when initializing
// message transport - it is set to `null`.
var eng = Engine.newBuilder().allowExperimentalOptions(true).options(engineOptions);
if (messageTransport != null) {
eng.serverTransport(messageTransport);
}
builder.engine(eng.build());
} else if (messageTransport != null) {
builder.serverTransport(messageTransport);
}

var ctx = builder.build();
ContextInsightSetup.configureContext(ctx);
return ctx;
Expand Down
1 change: 0 additions & 1 deletion engine/language-server/src/main/java/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
requires org.enso.task.progress.notifications;
requires org.enso.ydoc.polyfill;

exports org.enso.languageserver.boot;
exports org.enso.languageserver.filemanager to scala.library;
exports org.enso.languageserver.runtime to scala.library;
exports org.enso.languageserver.search to scala.library;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.slf4j.event.Level;
import scala.concurrent.ExecutionContext;

@org.openide.util.lookup.ServiceProvider(service = LanguageServerApi.class)
public final class LanguageServerRunner extends LanguageServerApi {
public LanguageServerRunner() {}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
package org.enso.languageserver.boot.resource;

import akka.event.EventStream;
import java.io.IOException;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.util.concurrent.*;
import org.apache.commons.io.FileUtils;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executor;
import java.util.concurrent.Semaphore;
import org.enso.languageserver.data.ProjectDirectoriesConfig;
import org.enso.languageserver.event.InitializedEvent;
import org.enso.logger.masking.MaskedPath;
import org.enso.searcher.memory.InMemorySuggestionsRepo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -72,14 +69,18 @@ public CompletableFuture<Void> init() {
}

private CompletableFuture<Void> initSuggestionsRepo() {
return CompletableFuture.runAsync(
() -> logger.debug("Initializing suggestions repo [{}]...", suggestionsRepo), executor)
.thenComposeAsync(
v -> {
if (!isInitialized)
return doInitSuggestionsRepo()
.exceptionallyComposeAsync(this::recoverInitializationError, executor);
else return CompletableFuture.completedFuture(v);
return CompletableFuture.supplyAsync(
() -> {
logger.debug("Initializing Suggestions repo [{}]...", suggestionsRepo);
try {
lock.acquire();
if (!isInitialized)
return doInitSuggestionsRepo()
.exceptionallyComposeAsync(this::recoverInitializationError, executor);
else return CompletableFuture.completedFuture(null);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
},
executor)
.thenRunAsync(
Expand All @@ -105,60 +106,6 @@ private CompletableFuture<Void> recoverInitializationError(Throwable error) {
.thenComposeAsync(v -> doInitSuggestionsRepo(), executor);
}

private CompletableFuture<Void> clearDatabaseFile(int retries) {
return CompletableFuture.runAsync(
() -> {
if (!isInitialized) {
logger.debug("Clear database file. Attempt #{}", retries + 1);
try {
Files.delete(projectDirectoriesConfig.suggestionsDatabaseFile().toPath());
} catch (IOException e) {
throw new CompletionException(e);
}
}
},
executor)
.exceptionallyComposeAsync(error -> recoverClearDatabaseFile(error, retries), executor);
}

private CompletableFuture<Void> recoverClearDatabaseFile(Throwable error, int retries) {
if (error instanceof CompletionException) {
return recoverClearDatabaseFile(error.getCause(), retries);
} else if (error instanceof NoSuchFileException) {
logger.warn(
"Failed to delete the database file. Attempt #{}. File does not exist [{}]",
retries + 1,
new MaskedPath(projectDirectoriesConfig.suggestionsDatabaseFile().toPath()));
return CompletableFuture.completedFuture(null);
} else if (error instanceof FileSystemException) {
logger.error(
"Failed to delete the database file. Attempt #{}. The file will be removed during the"
+ " shutdown",
retries + 1,
error);
Runtime.getRuntime()
.addShutdownHook(
new Thread(
() ->
FileUtils.deleteQuietly(projectDirectoriesConfig.suggestionsDatabaseFile())));
return CompletableFuture.failedFuture(error);
} else if (error instanceof IOException) {
logger.error("Failed to delete the database file. Attempt #{}", retries + 1, error);
if (retries < MAX_RETRIES) {
try {
Thread.sleep(RETRY_DELAY_MILLIS);
} catch (InterruptedException e) {
throw new CompletionException(e);
}
return clearDatabaseFile(retries + 1);
} else {
return CompletableFuture.failedFuture(error);
}
}

return CompletableFuture.completedFuture(null);
}

private CompletionStage<Void> doInitSuggestionsRepo() {
return FutureConverters.asJava(suggestionsRepo.init()).thenAcceptAsync(res -> {}, executor);
}
Expand Down
Loading

0 comments on commit 7cd1853

Please sign in to comment.