feat(core, jdbc): introduce a JDBC indexer

cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java

@@ -70,6 +70,9 @@ public class StandAloneCommand extends AbstractServerCommand {
     @CommandLine.Option(names = {"--not-start-executors"}, split=",", description = "a list of Kafka Stream executors to not start, separated by a command. Use it only with the Kafka queue, for debugging purpose.")
     private List<String> notStartExecutors = Collections.emptyList();
 
+    @CommandLine.Option(names = {"--no-indexer"}, description = "Flag to disable starting an embedded indexer.")
+    boolean indexerDisabled = false;
+
     @Override
     public boolean isFlowAutoLoadEnabled() {
         return !tutorialsDisabled;
@@ -111,6 +114,10 @@ public Integer call() throws Exception {
             standAloneRunner.setWorkerThread(this.workerThread);
         }
 
+        if (this.indexerDisabled) {
+            standAloneRunner.setIndexerEnabled(false);
+        }
+
         standAloneRunner.run();
 
         if (fileWatcher != null) {

cli/src/main/java/io/kestra/cli/commands/servers/WebServerCommand.java

@@ -3,27 +3,38 @@
 import com.google.common.collect.ImmutableMap;
 import io.kestra.core.contexts.KestraContext;
 import io.kestra.core.models.ServerType;
+import io.kestra.core.runners.ExecutorInterface;
+import io.kestra.core.runners.IndexerInterface;
 import io.kestra.core.utils.Await;
+import io.kestra.core.utils.ExecutorsUtils;
 import io.micronaut.context.ApplicationContext;
 import jakarta.inject.Inject;
 import lombok.extern.slf4j.Slf4j;
 import picocli.CommandLine;
 import picocli.CommandLine.Option;
 
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
 
 @CommandLine.Command(
     name = "webserver",
     description = "start the webserver"
 )
 @Slf4j
 public class WebServerCommand extends AbstractServerCommand {
+    private ExecutorService poolExecutor;
+
     @Inject
     private ApplicationContext applicationContext;
 
+    @Inject
+    private ExecutorsUtils executorsUtils;
+
     @Option(names = {"--no-tutorials"}, description = "Flag to disable auto-loading of tutorial flows.")
     boolean tutorialsDisabled = false;
 
+    @Option(names = {"--no-indexer"}, description = "Flag to disable starting an embedded indexer.")
+    boolean indexerDisabled = false;
 
     @Override
     public boolean isFlowAutoLoadEnabled() {
@@ -40,9 +51,26 @@ public static Map<String, Object> propertiesOverrides() {
     @Override
     public Integer call() throws Exception {
         super.call();
+
+        // start the indexer
+        if (!indexerDisabled) {
+            log.info("Starting an embedded indexer, this can be disabled by using `--no-indexer`.");
+            poolExecutor = executorsUtils.cachedThreadPool("webserver-indexer");
+            poolExecutor.execute(applicationContext.getBean(IndexerInterface.class));
+        }
+
         log.info("Webserver started");
-        this.shutdownHook(() -> KestraContext.getContext().shutdown());
+        this.shutdownHook(() -> {
+            this.close();
+            KestraContext.getContext().shutdown();
+        });
         Await.until(() -> !this.applicationContext.isRunning());
         return 0;
     }
+
+    private void close() {
+        if (this.poolExecutor != null) {
+            this.poolExecutor.shutdown();
+        }
+    }
 }

core/src/main/java/io/kestra/core/repositories/SaveRepositoryInterface.java

@@ -1,5 +1,11 @@
 package io.kestra.core.repositories;
 
-public interface SaveRepositoryInterface <T> {
-    T save(T flow);
+import java.util.List;
+
+public interface SaveRepositoryInterface<T> {
+    T save(T item);
+
+    default int saveBatch(List<T> items) {
+        throw new UnsupportedOperationException();
+    }
 }

core/src/main/java/io/kestra/core/runners/Indexer.java

@@ -1,133 +1,4 @@
 package io.kestra.core.runners;
-
-import io.kestra.core.metrics.MetricRegistry;
-import io.kestra.core.models.executions.Execution;
-import io.kestra.core.models.executions.LogEntry;
-import io.kestra.core.models.executions.MetricEntry;
-import io.kestra.core.queues.QueueFactoryInterface;
-import io.kestra.core.queues.QueueInterface;
-import io.kestra.core.repositories.ExecutionRepositoryInterface;
-import io.kestra.core.repositories.LogRepositoryInterface;
-import io.kestra.core.repositories.MetricRepositoryInterface;
-import io.kestra.core.repositories.SaveRepositoryInterface;
-import io.kestra.core.repositories.TriggerRepositoryInterface;
-import io.kestra.core.server.ServiceStateChangeEvent;
-import io.kestra.core.utils.IdUtils;
-import io.micronaut.context.annotation.Requires;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
-import io.micronaut.context.event.ApplicationEventPublisher;
-import jakarta.annotation.PreDestroy;
-import jakarta.inject.Inject;
-import jakarta.inject.Named;
-import jakarta.inject.Singleton;
-import lombok.extern.slf4j.Slf4j;
-
-@SuppressWarnings("this-escape")
-@Slf4j
-@Singleton
-@Requires(beans = {ExecutionRepositoryInterface.class, LogRepositoryInterface.class, TriggerRepositoryInterface.class})
-public class Indexer implements IndexerInterface {
-    private final ExecutionRepositoryInterface executionRepository;
-    private final QueueInterface<Execution> executionQueue;
-    private final LogRepositoryInterface logRepository;
-    private final QueueInterface<LogEntry> logQueue;
-
-    private final MetricRepositoryInterface metricRepository;
-    private final QueueInterface<MetricEntry> metricQueue;
-    private final MetricRegistry metricRegistry;
-    private final List<Runnable> receiveCancellations = new ArrayList<>();
-
-    private final String id = IdUtils.create();
-    private final AtomicReference<ServiceState> state = new AtomicReference<>();
-    private final ApplicationEventPublisher<ServiceStateChangeEvent> eventPublisher;
-
-    @Inject
-    public Indexer(
-        ExecutionRepositoryInterface executionRepository,
-        @Named(QueueFactoryInterface.EXECUTION_NAMED) QueueInterface<Execution> executionQueue,
-        LogRepositoryInterface logRepository,
-        @Named(QueueFactoryInterface.WORKERTASKLOG_NAMED) QueueInterface<LogEntry> logQueue,
-        MetricRepositoryInterface metricRepositor,
-        @Named(QueueFactoryInterface.METRIC_QUEUE) QueueInterface<MetricEntry> metricQueue,
-        MetricRegistry metricRegistry,
-        ApplicationEventPublisher<ServiceStateChangeEvent> eventPublisher
-    ) {
-        this.executionRepository = executionRepository;
-        this.executionQueue = executionQueue;
-        this.logRepository = logRepository;
-        this.logQueue = logQueue;
-        this.metricRepository = metricRepositor;
-        this.metricQueue = metricQueue;
-        this.metricRegistry = metricRegistry;
-        this.eventPublisher = eventPublisher;
-        setState(ServiceState.CREATED);
-    }
-
-    @Override
-    public void run() {
-        this.send(executionQueue, executionRepository);
-        this.send(logQueue, logRepository);
-        this.send(metricQueue, metricRepository);
-        setState(ServiceState.RUNNING);
-    }
-
-    protected <T> void send(QueueInterface<T> queueInterface, SaveRepositoryInterface<T> saveRepositoryInterface) {
-        this.receiveCancellations.addFirst(queueInterface.receive(Indexer.class, either -> {
-            if (either.isRight()) {
-                log.error("unable to deserialize an item: {}", either.getRight().getMessage());
-                return;
-            }
-
-            T item = either.getLeft();
-            this.metricRegistry.counter(MetricRegistry.METRIC_INDEXER_REQUEST_COUNT, "type", item.getClass().getName()).increment();
-            this.metricRegistry.counter(MetricRegistry.METRIC_INDEXER_MESSAGE_IN_COUNT, "type", item.getClass().getName()).increment();
-
-            this.metricRegistry.timer(MetricRegistry.METRIC_INDEXER_REQUEST_DURATION, "type", item.getClass().getName()).record(() -> {
-                saveRepositoryInterface.save(item);
-                this.metricRegistry.counter(MetricRegistry.METRIC_INDEXER_MESSAGE_OUT_COUNT, "type", item.getClass().getName()).increment();
-            });
-        }));
-    }
-
-    protected void setState(final ServiceState state) {
-        this.state.set(state);
-        this.eventPublisher.publishEvent(new ServiceStateChangeEvent(this));
-    }
-
-    /** {@inheritDoc} **/
-    @Override
-    public String getId() {
-        return id;
-    }
-    /** {@inheritDoc} **/
-    @Override
-    public ServiceType getType() {
-        return ServiceType.INDEXER;
-    }
-    /** {@inheritDoc} **/
-    @Override
-    public ServiceState getState() {
-        return state.get();
-    }
-
-    @PreDestroy
-    @Override
-    public void close() {
-        setState(ServiceState.TERMINATING);
-        this.receiveCancellations.forEach(Runnable::run);
-        try {
-            this.executionQueue.close();
-            this.logQueue.close();
-            this.metricQueue.close();
-            setState(ServiceState.TERMINATED_GRACEFULLY);
-        } catch (IOException e) {
-            log.error("Failed to close the queue", e);
-            setState(ServiceState.TERMINATED_FORCED);
-        }
-    }
+// NOTE: this class is not used anymore but must be kept as it is used in as queue consumer both in JDBC and Kafka
+public class Indexer {
 }

core/src/main/java/io/kestra/core/runners/StandAloneRunner.java

@@ -29,6 +29,7 @@ public class StandAloneRunner implements RunnerInterface, AutoCloseable {
     @Setter protected int workerThread = Math.max(3, Runtime.getRuntime().availableProcessors());
     @Setter protected boolean schedulerEnabled = true;
     @Setter protected boolean workerEnabled = true;
+    @Setter protected boolean indexerEnabled = true;
 
     @Inject
     private ExecutorsUtils executorsUtils;
@@ -65,7 +66,7 @@ public void run() {
             servers.add(scheduler);
         }
 
-        if (applicationContext.containsBean(IndexerInterface.class)) {
+        if (indexerEnabled) {
             IndexerInterface indexer = applicationContext.getBean(IndexerInterface.class);
             poolExecutor.execute(indexer);
             servers.add(indexer);

jdbc-h2/src/main/java/io/kestra/repository/h2/H2Repository.java

@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import jakarta.annotation.Nullable;
 
@@ -43,21 +44,38 @@ public H2Repository(@Parameter JdbcTableConfig jdbcTableConfig,
     public void persist(T entity, DSLContext context, @Nullable Map<Field<Object>, Object> fields) {
         Map<Field<Object>, Object> finalFields = fields == null ? this.persistFields(entity) : fields;
 
+        this.persistInternal(entity, context, finalFields);
+    }
+
+    private int persistInternal(T entity, DSLContext context, Map<Field<Object>, Object> fields) {
         int affectedRows = context
             .update(table)
-            .set(finalFields)
+            .set(fields)
             .where(AbstractJdbcRepository.field("key").eq(key(entity)))
             .execute();
 
         if (affectedRows == 0) {
-            context
+           return  context
                 .insertInto(table)
                 .set(AbstractJdbcRepository.field("key"), key(entity))
-                .set(finalFields)
+                .set(fields)
                 .execute();
+        } else {
+            return affectedRows;
         }
     }
 
+    @Override
+    public int persistBatch(List<T> items) {
+        return dslContextWrapper.transactionResult(configuration -> {
+            DSLContext dslContext = DSL.using(configuration);
+            return items.stream()
+                .map(item -> this.persistInternal(item, dslContext, this.persistFields(item)))
+                .mapToInt(i -> i)
+                .sum();
+        });
+    }
+
     public Condition fullTextCondition(List<String> fields, String query) {
         if (query == null || query.equals("*")) {
             return DSL.trueCondition();

jdbc-postgres/src/main/java/io/kestra/repository/postgres/PostgresRepository.java

@@ -22,6 +22,7 @@
 import org.jooq.SelectConditionStep;
 import org.jooq.impl.DSL;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -75,6 +76,27 @@ public void persist(T entity, DSLContext context, @Nullable  Map<Field<Object>,
             .execute();
     }
 
+    @Override
+    public int persistBatch(List<T> items) {
+        return dslContextWrapper.transactionResult(configuration -> {
+            DSLContext dslContext = DSL.using(configuration);
+            var inserts = items.stream().map(item -> {
+                    Map<Field<Object>, Object> finalFields = this.persistFields(item);
+
+                    return dslContext
+                        .insertInto(table)
+                        .set(AbstractJdbcRepository.field("key"), key(item))
+                        .set(finalFields)
+                        .onConflict(AbstractJdbcRepository.field("key"))
+                        .doUpdate()
+                        .set(finalFields);
+                })
+                .toList();
+
+            return Arrays.stream(dslContext.batch(inserts).execute()).sum();
+        });
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public <R extends Record, E> ArrayListTotal<E> fetchPage(DSLContext context, SelectConditionStep<R> select, Pageable pageable, RecordMapper<R, E> mapper) {

jdbc/src/main/java/io/kestra/jdbc/AbstractJdbcRepository.java

@@ -96,6 +96,25 @@ public void persist(T entity, DSLContext dslContext, Map<Field<Object>, Object>
             .execute();
     }
 
+    public int persistBatch(List<T> items) {
+        return dslContextWrapper.transactionResult(configuration -> {
+            DSLContext dslContext = DSL.using(configuration);
+            var inserts = items.stream().map(item -> {
+                    Map<Field<Object>, Object> finalFields = this.persistFields(item);
+
+                    return dslContext
+                        .insertInto(table)
+                        .set(io.kestra.jdbc.repository.AbstractJdbcRepository.field("key"), key(item))
+                        .set(finalFields)
+                        .onDuplicateKeyUpdate()
+                        .set(finalFields);
+                })
+                .toList();
+
+            return Arrays.stream(dslContext.batch(inserts).execute()).sum();
+        });
+    }
+
     public int delete(T entity) {
         return dslContextWrapper.transactionResult(configuration -> {
             return this.delete(DSL.using(configuration), entity);