From a8d8e0140a71be3b74388c65acfd47d4a2d7bd1f Mon Sep 17 00:00:00 2001 From: Lasse Westh-Nielsen Date: Fri, 15 Dec 2023 15:45:48 +0100 Subject: [PATCH] migrate pathfinding algorithms to being reusable add a procedure facade for pathfinding add a new module for pathfinding algorithms under the applications hierarchy add a business facade in there for pathfinding algorithms add an algorithms-level facade for pathfinding algorithms; the difference between these last two is that the former does convenience and reuse; the latter is pure, pared back graph-to-result computation functionality --- .../java/org/neo4j/gds/paths/astar/AStar.java | 22 +- .../neo4j/gds/paths/dijkstra/Dijkstra.java | 63 +++- .../algorithms/path-finding/build.gradle | 27 ++ .../pathfinding/AlgorithmComputation.java | 26 ++ .../AlgorithmProcessingTemplate.java | 66 +++++ .../DefaultAlgorithmProcessingTemplate.java | 175 +++++++++++ .../pathfinding/DefaultMemoryGuard.java | 83 ++++++ .../algorithms/pathfinding/MemoryGuard.java | 58 ++++ .../pathfinding/MutateOrWriteStep.java | 32 ++ .../pathfinding/PathFindingAlgorithms.java | 143 +++++++++ .../PathFindingAlgorithmsFacade.java | 124 ++++++++ .../algorithms/pathfinding/ResultBuilder.java | 83 ++++++ .../pathfinding/ShortestPathMutateStep.java | 75 +++++ ...efaultAlgorithmProcessingTemplateTest.java | 280 ++++++++++++++++++ .../pathfinding/DefaultMemoryGuardTest.java | 118 ++++++++ .../pathfinding/ExampleConfiguration.java | 42 +++ .../neo4j/gds/core/utils/mem/GcListener.java | 2 +- .../core/utils/mem/GcListenerExtension.java | 33 +-- .../gds/core/utils/mem/HotSpotGcListener.java | 11 +- .../gds/core/utils/mem/GcListenerTest.java | 8 +- .../java/org/neo4j/gds/mem/MemoryGauge.java | 38 +++ .../catalog/GraphWriteNodePropertiesProc.java | 3 - ...TieredHarmonicCentralityWriteProcTest.java | 1 + .../HarmonicCentralityWriteProcTest.java | 1 + .../k1coloring/K1ColoringStreamProcTest.java | 1 + .../LabelPropagationMutateProcTest.java | 4 + .../ModularityOptimizationMutateProcTest.java | 1 + .../org/neo4j/gds/wcc/WccMutateProcTest.java | 4 + .../org/neo4j/gds/wcc/WccStatsProcTest.java | 1 + .../org/neo4j/gds/wcc/WccWriteProcTest.java | 1 + ....java => PathFindingMutateResultTest.java} | 2 +- proc/path-finding/build.gradle | 1 + .../ShortestPathMutateResultConsumer.java | 5 +- .../ShortestPathStreamResultConsumer.java | 7 +- .../longestPath/DagLongestPathStreamProc.java | 4 +- .../longestPath/DagLongestPathStreamSpec.java | 6 +- .../AllShortestPathsDeltaMutateProc.java | 4 +- .../AllShortestPathsDeltaMutateSpec.java | 6 +- .../AllShortestPathsDeltaStreamProc.java | 4 +- .../AllShortestPathsDeltaStreamSpec.java | 6 +- .../AllShortestPathsDijkstraMutateProc.java | 18 +- .../AllShortestPathsDijkstraMutateSpec.java | 6 +- .../AllShortestPathsDijkstraStreamProc.java | 18 +- .../AllShortestPathsDijkstraStreamSpec.java | 6 +- .../ShortestPathAStarMutateProc.java | 4 +- .../ShortestPathAStarMutateSpec.java | 6 +- .../ShortestPathAStarStreamProc.java | 14 +- .../ShortestPathAStarStreamSpec.java | 6 +- .../ShortestPathDijkstraMutateProc.java | 4 +- .../ShortestPathDijkstraMutateSpec.java | 6 +- .../ShortestPathDijkstraStreamProc.java | 14 +- .../ShortestPathDijkstraStreamSpec.java | 6 +- .../ShortestPathYensMutateProc.java | 4 +- .../ShortestPathYensMutateSpec.java | 6 +- .../ShortestPathYensStreamProc.java | 4 +- .../ShortestPathYensStreamSpec.java | 6 +- .../BfsMutateComputationResultConsumer.java | 6 +- .../gds/paths/traverse/BfsMutateProc.java | 4 +- .../gds/paths/traverse/BfsMutateSpec.java | 6 +- .../DfsMutateComputationResultConsumer.java | 6 +- .../gds/paths/traverse/DfsMutateProc.java | 4 +- .../gds/paths/traverse/DfsMutateSpec.java | 6 +- .../ShortestPathAStarStreamProcTest.java | 4 +- .../ShortestPathDijkstraStreamProcTest.java | 4 +- .../ShortestPathYensStreamProcTest.java | 8 +- procedures/extension/build.gradle | 1 + .../OpenGraphDataScienceExtension.java | 15 +- procedures/facade/build.gradle | 1 + .../gds/procedures/GraphDataScience.java | 10 +- .../pathfinding/AlgorithmHandle.java | 31 ++ .../pathfinding/PathFindingMutateResult.java | 14 +- .../PathFindingProcedureFacade.java | 176 +++++++++++ ...PathFindingResultBuilderForMutateMode.java | 52 ++++ ...PathFindingResultBuilderForStreamMode.java | 99 +++++++ .../pathfinding/PathFindingStreamResult.java | 12 +- procedures/integration/build.gradle | 2 + .../AlgorithmFacadeProviderFactory.java | 45 ++- .../AlgorithmProcedureFacadeProvider.java | 59 +++- .../integration/ExtensionBuilder.java | 82 +++-- .../integration}/GcListenerInstaller.java | 13 +- .../integration/GraphDataScienceProvider.java | 3 + settings.gradle | 3 + 82 files changed, 2145 insertions(+), 220 deletions(-) create mode 100644 applications/algorithms/path-finding/build.gradle create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmComputation.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmProcessingTemplate.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplate.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuard.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MemoryGuard.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MutateOrWriteStep.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithms.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithmsFacade.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ResultBuilder.java create mode 100644 applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ShortestPathMutateStep.java create mode 100644 applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplateTest.java create mode 100644 applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuardTest.java create mode 100644 applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/ExampleConfiguration.java create mode 100644 memory-usage/src/main/java/org/neo4j/gds/mem/MemoryGauge.java rename proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/{MutateResultTest.java => PathFindingMutateResultTest.java} (97%) create mode 100644 procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/AlgorithmHandle.java rename proc/path-finding/src/main/java/org/neo4j/gds/paths/MutateResult.java => procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingMutateResult.java (82%) create mode 100644 procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingProcedureFacade.java create mode 100644 procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForMutateMode.java create mode 100644 procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForStreamMode.java rename proc/path-finding/src/main/java/org/neo4j/gds/paths/StreamResult.java => procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingStreamResult.java (91%) rename {core/src/main/java/org/neo4j/gds/core/utils/mem => procedures/integration/src/main/java/org/neo4j/gds/procedures/integration}/GcListenerInstaller.java (92%) diff --git a/algo/src/main/java/org/neo4j/gds/paths/astar/AStar.java b/algo/src/main/java/org/neo4j/gds/paths/astar/AStar.java index d37c157c630..a38d654409d 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/astar/AStar.java +++ b/algo/src/main/java/org/neo4j/gds/paths/astar/AStar.java @@ -27,6 +27,7 @@ import org.neo4j.gds.paths.astar.config.ShortestPathAStarBaseConfig; import org.neo4j.gds.paths.dijkstra.Dijkstra; import org.neo4j.gds.paths.dijkstra.PathFindingResult; +import org.neo4j.gds.termination.TerminationFlag; import java.util.Optional; @@ -36,16 +37,29 @@ public final class AStar extends Algorithm { private final Dijkstra dijkstra; - private AStar(Dijkstra dijkstra) { + private AStar(Dijkstra dijkstra, TerminationFlag terminationFlag) { super(dijkstra.getProgressTracker()); this.dijkstra = dijkstra; - this.terminationFlag = dijkstra.getTerminationFlag(); + this.terminationFlag = terminationFlag; } + /** + * @deprecated Use the one with termination flag + */ + @Deprecated public static AStar sourceTarget( Graph graph, ShortestPathAStarBaseConfig config, ProgressTracker progressTracker + ) { + return sourceTarget(graph, config, progressTracker, TerminationFlag.RUNNING_TRUE); + } + + public static AStar sourceTarget( + Graph graph, + ShortestPathAStarBaseConfig config, + ProgressTracker progressTracker, + TerminationFlag terminationFlag ) { var latitudeProperty = config.latitudeProperty(); var longitudeProperty = config.longitudeProperty(); @@ -70,8 +84,8 @@ public static AStar sourceTarget( var heuristic = new HaversineHeuristic(latitudeProperties, longitudeProperties, targetNode); // Init dijkstra algorithm for computing shortest paths - var dijkstra = Dijkstra.sourceTarget(graph, config, Optional.of(heuristic), progressTracker); - return new AStar(dijkstra); + var dijkstra = Dijkstra.sourceTarget(graph, config, Optional.of(heuristic), progressTracker, terminationFlag); + return new AStar(dijkstra, terminationFlag); } diff --git a/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java b/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java index c70d7920c09..38f802f2b2b 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java +++ b/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java @@ -32,6 +32,7 @@ import org.neo4j.gds.paths.ImmutablePathResult; import org.neo4j.gds.paths.PathResult; import org.neo4j.gds.paths.SourceTargetShortestPathBaseConfig; +import org.neo4j.gds.termination.TerminationFlag; import java.util.Optional; import java.util.function.LongToDoubleFunction; @@ -71,53 +72,94 @@ public final class Dijkstra extends Algorithm { private RelationshipFilter relationshipFilter = (sourceId, targetId, relationshipId) -> true; /** - * Configure Dijkstra to compute at most one source-target shortest path. + * @deprecated Use the other one, with termination flag */ + @Deprecated public static Dijkstra sourceTarget( Graph graph, SourceTargetShortestPathBaseConfig config, Optional heuristicFunction, ProgressTracker progressTracker ) { - long sourceNode = graph.toMappedNodeId(config.sourceNode()); - long targetNode = graph.toMappedNodeId(config.targetNode()); + return sourceTarget( + graph, + config, + heuristicFunction, + progressTracker, + TerminationFlag.RUNNING_TRUE + ); + } + + /** + * Configure Dijkstra to compute at most one source-target shortest path. + */ + public static Dijkstra sourceTarget( + Graph graph, + SourceTargetShortestPathBaseConfig configuration, + Optional heuristicFunction, + ProgressTracker progressTracker, + TerminationFlag terminationFlag + ) { + long sourceNode = graph.toMappedNodeId(configuration.sourceNode()); + long targetNode = graph.toMappedNodeId(configuration.targetNode()); return new Dijkstra( graph, sourceNode, node -> node == targetNode ? EMIT_AND_STOP : CONTINUE, - config.trackRelationships(), + configuration.trackRelationships(), heuristicFunction, - progressTracker + progressTracker, + terminationFlag ); } /** - * Configure Dijkstra to compute all single-source shortest path. + * @deprecated Use the other one with termination flag */ public static Dijkstra singleSource( Graph graph, AllShortestPathsBaseConfig config, Optional heuristicFunction, ProgressTracker progressTracker + ) { + return singleSource( + graph, + config, + heuristicFunction, + progressTracker, + TerminationFlag.RUNNING_TRUE + ); + } + + /** + * Configure Dijkstra to compute all single-source shortest path. + */ + public static Dijkstra singleSource( + Graph graph, + AllShortestPathsBaseConfig config, + Optional heuristicFunction, + ProgressTracker progressTracker, + TerminationFlag terminationFlag ) { return new Dijkstra(graph, graph.toMappedNodeId(config.sourceNode()), node -> EMIT_AND_CONTINUE, config.trackRelationships(), heuristicFunction, - progressTracker + progressTracker, + terminationFlag ); } - private Dijkstra( + public Dijkstra( Graph graph, long sourceNode, TraversalPredicate traversalPredicate, boolean trackRelationships, Optional heuristicFunction, - ProgressTracker progressTracker - ) { + ProgressTracker progressTracker, + TerminationFlag terminationFlag) { super(progressTracker); this.graph = graph; this.sourceNode = sourceNode; @@ -131,6 +173,7 @@ private Dijkstra( this.relationships = trackRelationships ? new HugeLongLongMap() : null; this.visited = new BitSet(); this.pathIndex = 0L; + this.terminationFlag = terminationFlag; } public Dijkstra withSourceNode(long sourceNode) { diff --git a/applications/algorithms/path-finding/build.gradle b/applications/algorithms/path-finding/build.gradle new file mode 100644 index 00000000000..5ffe838dcde --- /dev/null +++ b/applications/algorithms/path-finding/build.gradle @@ -0,0 +1,27 @@ +apply plugin: 'java-library' + +description = 'Neo4j Graph Data Science :: Path Finding Algorithms' + +group = 'org.neo4j.gds' + +dependencies { + compileOnly(group: 'org.neo4j', name: 'neo4j-logging', version: ver.'neo4j') { transitive = false } + + implementation project(':algo') + implementation project(':algo-common') + implementation project(':config-api') + implementation project(':core') + implementation project(':logging') + implementation project(':memory-usage') + implementation project(':metrics-api') + implementation project(':progress-tracking') + implementation project(':string-formatting') + implementation project(':termination') + + testImplementation group: 'org.assertj', name: 'assertj-core', version: ver.'assertj' + testImplementation platform(dep.junit5bom) + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: ver.'junit5bom' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: ver.'junit5bom' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: ver.'junit5bom' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: ver.'mockito-junit-jupiter' +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmComputation.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmComputation.java new file mode 100644 index 00000000000..a57d6bdc295 --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmComputation.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.Graph; + +public interface AlgorithmComputation { + RESULT compute(Graph graph); +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmProcessingTemplate.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmProcessingTemplate.java new file mode 100644 index 00000000000..c4967bd6b29 --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/AlgorithmProcessingTemplate.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.GraphName; +import org.neo4j.gds.config.AlgoBaseConfig; +import org.neo4j.gds.config.RelationshipWeightConfig; +import org.neo4j.gds.core.utils.mem.MemoryEstimation; + +import java.util.Optional; +import java.util.function.Supplier; + +/** + * So, internally, all (pathfinding) algorithms in all modes follow these same steps: + *
    + *
  1. You load a graph + *
  2. You validate the algorithms vs the graph (is it empty, memory usage, etc.) + *
  3. You establish timings and metrics + *
  4. YOU CALL THE ACTUAL ALGORITHM! Minor detail... + *
  5. Next up, mutates or writes, the possible side effect steps + *
  6. Lastly result rendering + *
+ * Result rendering is a whole chapter. It is independent of where call came from, so injected. + * It can use any of these parameters: + *
    + *
  • graph + *
  • graph store + *
  • configuration + *
  • timings, including side effect steps + *
+ * Injected, plus mode-conditional: it's a wide parameter list... + * There is always a result. For streaming, it is duh; for the others it is metadata. + * Side effect steps are optional, and they are injected in some sense, but also _selected_. + * Because luckily those behaviours are generic enough that you can just select them from a catalogue. + * So we _instrument_ with zero or one mode behaviour selected from a catalogue, + * and _instrument_ with bespoke result rendering. + */ +public interface AlgorithmProcessingTemplate { + + RESULT_TO_CALLER processAlgorithm( + GraphName graphName, + CONFIGURATION configuration, + String humanReadableAlgorithmName, + Supplier estimationFactory, + AlgorithmComputation algorithmComputation, + Optional> mutateOrWriteStep, + ResultBuilder resultBuilder + ); +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplate.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplate.java new file mode 100644 index 00000000000..032aefae61f --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplate.java @@ -0,0 +1,175 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.apache.commons.lang3.tuple.Pair; +import org.neo4j.gds.api.DatabaseId; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.GraphName; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.api.User; +import org.neo4j.gds.config.AlgoBaseConfig; +import org.neo4j.gds.config.RelationshipWeightConfig; +import org.neo4j.gds.core.loading.GraphStoreCatalogService; +import org.neo4j.gds.core.utils.ProgressTimer; +import org.neo4j.gds.core.utils.mem.MemoryEstimation; +import org.neo4j.gds.logging.Log; +import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService; + +import java.util.Optional; +import java.util.function.Supplier; + +public class DefaultAlgorithmProcessingTemplate implements AlgorithmProcessingTemplate { + // global dependencies + private final Log log; + private final AlgorithmMetricsService algorithmMetricsService; + private final GraphStoreCatalogService graphStoreCatalogService; + private final MemoryGuard memoryGuard; + + // request scoped parameters + private final DatabaseId databaseId; + private final User user; + + public DefaultAlgorithmProcessingTemplate( + Log log, + AlgorithmMetricsService algorithmMetricsService, + GraphStoreCatalogService graphStoreCatalogService, + MemoryGuard memoryGuard, + DatabaseId databaseId, + User user + ) { + this.log = log; + this.algorithmMetricsService = algorithmMetricsService; + this.graphStoreCatalogService = graphStoreCatalogService; + this.databaseId = databaseId; + this.user = user; + this.memoryGuard = memoryGuard; + } + + @Override + public RESULT_TO_CALLER processAlgorithm( + GraphName graphName, + CONFIGURATION configuration, + String humanReadableAlgorithmName, + Supplier estimationFactory, + AlgorithmComputation algorithmComputation, + Optional> mutateOrWriteStep, + ResultBuilder resultBuilder + ) { + Pair graphWithGraphStore = graphLoadAndValidationWithTiming( + graphName, + configuration, + resultBuilder + ); + + var graph = graphWithGraphStore.getLeft(); + var graphStore = graphWithGraphStore.getRight(); + + if (graph.isEmpty()) return resultBuilder.build(graph, graphStore, Optional.empty()); + + memoryGuard.assertAlgorithmCanRun(humanReadableAlgorithmName, configuration, graph, estimationFactory); + + // do the actual computation + var result = computeWithTiming(humanReadableAlgorithmName, algorithmComputation, resultBuilder, graph); + + // do any side effects + mutateOrWriteWithTiming(mutateOrWriteStep, resultBuilder, graph, graphStore, result); + + // inject dependencies to render results + return resultBuilder.build(graph, graphStore, Optional.ofNullable(result)); + } + + /** + * To fully generalise this out from pathfinding, there are two issues to solve here: + * + *
    + *
  • Having to have configurations inherit from both AlgoBaseConfig and RelationshipWeightConfig is not good. + * The stipulation for RelationshipWeightConfig could be solved with a conditional, or by lifting relationship weights up as a first class thing. + * (We can have a longer talk about configurations inheritance another time)
    • + *
  • ValidationConfiguration are a thing that is not used in path finding and so it is left out for now. + * Generally though there are hooks that are needed for validation: before loading, after loading, ... + *

    + * We can add that when needed as more instrumentation.

    • + *
+ */ + Pair graphLoadAndValidationWithTiming( + GraphName graphName, + CONFIGURATION configuration, + ResultBuilder resultBuilder + ) { + try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withPreProcessingMillis)) { + // tee up the graph we want to work on + var graphWithGraphStore = graphStoreCatalogService.getGraphWithGraphStore( + graphName, + configuration, + configuration.relationshipWeightProperty(), + user, + databaseId + ); + + // ValidationConfiguration post-load stuff would go here + + return graphWithGraphStore; + } + } + + RESULT_FROM_ALGORITHM computeWithTiming( + String humanReadableAlgorithmName, + AlgorithmComputation algorithmComputation, + ResultBuilder resultBuilder, + Graph graph + ) { + try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withComputeMillis)) { + return computeWithMetric(humanReadableAlgorithmName, algorithmComputation, graph); + } + } + + private RESULT_FROM_ALGORITHM computeWithMetric( + String humanReadableAlgorithmName, + AlgorithmComputation algorithmComputation, + Graph graph + ) { + var executionMetric = algorithmMetricsService.create(humanReadableAlgorithmName); + + try (executionMetric) { + executionMetric.start(); + + return algorithmComputation.compute(graph); + } catch (RuntimeException e) { + log.warn("computation failed, halting metrics gathering", e); + executionMetric.failed(); + throw e; + } + } + + void mutateOrWriteWithTiming( + Optional> mutateOrWriteStep, + ResultBuilder resultBuilder, + Graph graph, + GraphStore graphStore, + RESULT_FROM_ALGORITHM result + ) { + mutateOrWriteStep.ifPresent(step -> { + try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withPostProcessingMillis)) { + step.execute(graph, graphStore, result, resultBuilder); + } + }); + } +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuard.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuard.java new file mode 100644 index 00000000000..908623e07a1 --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuard.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.config.ConcurrencyConfig; +import org.neo4j.gds.core.GraphDimensions; +import org.neo4j.gds.core.utils.mem.MemoryEstimation; +import org.neo4j.gds.exceptions.MemoryEstimationNotImplementedException; +import org.neo4j.gds.logging.Log; +import org.neo4j.gds.mem.MemoryGauge; +import org.neo4j.gds.utils.StringFormatting; + +import java.util.function.Supplier; + +public class DefaultMemoryGuard implements MemoryGuard { + private final Log log; + private final boolean useMaxMemoryEstimation; + private final MemoryGauge memoryGauge; + + public DefaultMemoryGuard(Log log, boolean useMaxMemoryEstimation, MemoryGauge memoryGauge) { + this.log = log; + this.useMaxMemoryEstimation = useMaxMemoryEstimation; + this.memoryGauge = memoryGauge; + } + + @Override + public void assertAlgorithmCanRun( + String humanReadableAlgorithmName, + CONFIGURATION configuration, + Graph graph, + Supplier estimationFactory + ) throws IllegalStateException { + try { + var memoryEstimation = estimationFactory.get(); + + var graphDimensions = GraphDimensions.of(graph.nodeCount(), graph.relationshipCount()); + + var memoryTree = memoryEstimation.estimate(graphDimensions, configuration.concurrency()); + + var memoryRange = memoryTree.memoryUsage(); + + var bytesRequired = useMaxMemoryEstimation ? memoryRange.max : memoryRange.min; + + assertAlgorithmCanRun(humanReadableAlgorithmName, bytesRequired); + } catch (MemoryEstimationNotImplementedException e) { + log.info("Memory usage estimate not available for " + humanReadableAlgorithmName + ", skipping guard"); + } + } + + private void assertAlgorithmCanRun(String humanReadableAlgorithmName, long bytesRequired) + throws IllegalStateException { + long bytesAvailable = memoryGauge.availableMemory(); + + if (bytesRequired > bytesAvailable) { + var message = StringFormatting.formatWithLocale( + "Memory required to run %s (%db) exceeds available memory (%db)", + humanReadableAlgorithmName, + bytesRequired, + bytesAvailable + ); + + throw new IllegalStateException(message); + } + } +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MemoryGuard.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MemoryGuard.java new file mode 100644 index 00000000000..e94e187cce2 --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MemoryGuard.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.config.ConcurrencyConfig; +import org.neo4j.gds.core.utils.mem.MemoryEstimation; + +import java.util.function.Supplier; + +/** + * This is just memory guarding. Do not conflate with UI concerns. + */ +public interface MemoryGuard { + /** + * This could be handy for tests + */ + MemoryGuard DISABLED = new MemoryGuard() { + @Override + public void assertAlgorithmCanRun( + String humanReadableAlgorithmName, + CONFIGURATION configuration, + Graph graph, + Supplier estimationFactory + ) { + // do nothing + } + }; + + /** + * Measure how much memory is needed vs how much is available. + * + * @throws IllegalStateException when there is not enough memory available to run the algorithm in the given configuration on the given graph + */ + void assertAlgorithmCanRun( + String humanReadableAlgorithmName, + CONFIGURATION configuration, + Graph graph, + Supplier estimationFactory + ) throws IllegalStateException; +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MutateOrWriteStep.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MutateOrWriteStep.java new file mode 100644 index 00000000000..957994f4946 --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/MutateOrWriteStep.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.GraphStore; + +public interface MutateOrWriteStep { + void execute( + Graph graph, + GraphStore graphStore, + RESULT_FROM_ALGORITHM result, + ResultBuilder resultBuilder + ); +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithms.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithms.java new file mode 100644 index 00000000000..1056ec1cd41 --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithms.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.config.AlgoBaseConfig; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.Task; +import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.TaskTreeProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.Tasks; +import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory; +import org.neo4j.gds.logging.Log; +import org.neo4j.gds.paths.AllShortestPathsBaseConfig; +import org.neo4j.gds.paths.SourceTargetShortestPathBaseConfig; +import org.neo4j.gds.paths.astar.AStar; +import org.neo4j.gds.paths.astar.config.ShortestPathAStarBaseConfig; +import org.neo4j.gds.paths.dijkstra.Dijkstra; +import org.neo4j.gds.paths.dijkstra.PathFindingResult; +import org.neo4j.gds.termination.TerminationFlag; + +import java.util.Optional; + +/** + * Here is the bottom business facade for path finding (or top layer in another module, or maybe not even a facade, ...). + * As such, it is purely about calling algorithms and functional algorithm things. + * The layers above will do input validation and result shaping. + * For example, at this point we have stripped off modes. Modes are a result rendering or marshalling concept, + * where you _use_ the results computed here, and ETL them. + * Associated mode-specific validation is also done in layers above. + */ +public class PathFindingAlgorithms { + // global scoped dependencies + private final Log log; + + // request scoped parameters + private final TaskRegistryFactory taskRegistryFactory; + private final TerminationFlag terminationFlag; + private final UserLogRegistryFactory userLogRegistryFactory; + + public PathFindingAlgorithms( + Log log, + TaskRegistryFactory taskRegistryFactory, + TerminationFlag terminationFlag, + UserLogRegistryFactory userLogRegistryFactory + ) { + this.log = log; + this.taskRegistryFactory = taskRegistryFactory; + this.terminationFlag = terminationFlag; + this.userLogRegistryFactory = userLogRegistryFactory; + } + + PathFindingResult singlePairShortestPathAStar( + Graph graph, + ShortestPathAStarBaseConfig configuration + ) { + var progressTracker = createProgressTracker( + configuration, + Tasks.leaf("AStar", graph.relationshipCount()) + ); + + var algorithm = AStar.sourceTarget(graph, configuration, progressTracker, terminationFlag); + + return algorithm.compute(); + } + + /** + * This is conceptually very pretty, it speaks to you: call Dijkstra on a graph, return the result. + * Caller is responsible for looking up the graph, including the (implicit) validation that that involves. + * And they also get to do result rendering, using details of user request and GraphStore state. + * Down here though it is just the algorithm. + */ + PathFindingResult singlePairShortestPathDijkstra( + Graph graph, + SourceTargetShortestPathBaseConfig configuration + ) { + var progressTracker = createProgressTracker( + configuration, + Tasks.leaf("Dijkstra", graph.relationshipCount()) + ); + + var dijkstra = Dijkstra.sourceTarget(graph, configuration, Optional.empty(), progressTracker, terminationFlag); + + return dijkstra.compute(); + } + + PathFindingResult singleSourceShortestPathDijkstra( + Graph graph, + AllShortestPathsBaseConfig configuration + ) { + var progressTracker = createProgressTracker( + configuration, + Tasks.leaf("Dijkstra", graph.relationshipCount()) + ); + + var dijkstra = Dijkstra.singleSource(graph, configuration, Optional.empty(), progressTracker, terminationFlag); + + return dijkstra.compute(); + } + + private ProgressTracker createProgressTracker( + AlgoBaseConfig configuration, + Task progressTask + ) { + if (configuration.logProgress()) { + return new TaskProgressTracker( + progressTask, + (org.neo4j.logging.Log) log.getNeo4jLog(), + configuration.concurrency(), + configuration.jobId(), + taskRegistryFactory, + userLogRegistryFactory + ); + } + + return new TaskTreeProgressTracker( + progressTask, + (org.neo4j.logging.Log) log.getNeo4jLog(), + configuration.concurrency(), + configuration.jobId(), + taskRegistryFactory, + userLogRegistryFactory + ); + } +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithmsFacade.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithmsFacade.java new file mode 100644 index 00000000000..a00f496b70c --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithmsFacade.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.GraphName; +import org.neo4j.gds.paths.astar.AStarMemoryEstimateDefinition; +import org.neo4j.gds.paths.astar.config.ShortestPathAStarStreamConfig; +import org.neo4j.gds.paths.dijkstra.DijkstraMemoryEstimateDefinition; +import org.neo4j.gds.paths.dijkstra.PathFindingResult; +import org.neo4j.gds.paths.dijkstra.config.AllShortestPathsDijkstraMutateConfig; +import org.neo4j.gds.paths.dijkstra.config.AllShortestPathsDijkstraStreamConfig; +import org.neo4j.gds.paths.dijkstra.config.ShortestPathDijkstraStreamConfig; + +import java.util.Optional; + +/** + * Here is the top level business facade for all your path finding needs. + * It will have all pathfinding algorithms on it, each in four modes {mutate, stats, stream, write}, + * and algorithm estimates too. + *

+ * That could get really long so consider something cleverer, like per mode sub-facades, or algorithm sub-facades. + * Prolly the former because cohesion. + *

+ * Because this is the top level thing, we should make it very useful and usable, + * in the sense that we can capture reuse here that different UIs need. Neo4j Procedures coming in, maybe Arrow, + * let's make life easy for them. + *

+ * Concretely, we have UI layers inject result rendering as that is bespoke. + * We delegate downwards for the actual computations. + * But importantly, this is where we decide which, if any, mutate or write hooks need to be injected. + */ +public class PathFindingAlgorithmsFacade { + private final AlgorithmProcessingTemplate algorithmProcessingTemplate; + private final PathFindingAlgorithms pathFindingAlgorithms; + + public PathFindingAlgorithmsFacade( + AlgorithmProcessingTemplate algorithmProcessingTemplate, + PathFindingAlgorithms pathFindingAlgorithms + ) { + this.algorithmProcessingTemplate = algorithmProcessingTemplate; + this.pathFindingAlgorithms = pathFindingAlgorithms; + } + + public RESULT singlePairShortestPathAStarStream( + GraphName graphName, + ShortestPathAStarStreamConfig configuration, + ResultBuilder resultBuilder + ) { + return algorithmProcessingTemplate.processAlgorithm( + graphName, + configuration, + "AStar", + () -> new AStarMemoryEstimateDefinition().memoryEstimation(configuration), + graph -> pathFindingAlgorithms.singlePairShortestPathAStar(graph, configuration), + Optional.empty(), + resultBuilder + ); + } + + public RESULT singlePairShortestPathDijkstraStream( + GraphName graphName, + ShortestPathDijkstraStreamConfig configuration, + ResultBuilder resultBuilder + ) { + return algorithmProcessingTemplate.processAlgorithm( + graphName, + configuration, + "Dijkstra", + () -> new DijkstraMemoryEstimateDefinition().memoryEstimation(configuration), + graph -> pathFindingAlgorithms.singlePairShortestPathDijkstra(graph, configuration), + Optional.empty(), + resultBuilder + ); + } + + public RESULT singleSourceShortestPathDijkstraMutate( + GraphName graphName, + AllShortestPathsDijkstraMutateConfig configuration, + ResultBuilder resultBuilder + ) { + return algorithmProcessingTemplate.processAlgorithm( + graphName, + configuration, + "Dijkstra", + () -> new DijkstraMemoryEstimateDefinition().memoryEstimation(configuration), + graph -> pathFindingAlgorithms.singleSourceShortestPathDijkstra(graph, configuration), + Optional.of(new ShortestPathMutateStep(configuration)), + resultBuilder + ); + } + + public RESULT singleSourceShortestPathDijkstraStream( + GraphName graphName, + AllShortestPathsDijkstraStreamConfig configuration, + ResultBuilder resultBuilder + ) { + return algorithmProcessingTemplate.processAlgorithm( + graphName, + configuration, + "Dijkstra", + () -> new DijkstraMemoryEstimateDefinition().memoryEstimation(configuration), + graph -> pathFindingAlgorithms.singleSourceShortestPathDijkstra(graph, configuration), + Optional.empty(), + resultBuilder + ); + } +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ResultBuilder.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ResultBuilder.java new file mode 100644 index 00000000000..3cbea880c1a --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ResultBuilder.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.GraphStore; + +import java.util.Optional; + +/** + * This builder gathers data as part of algorithm processing. Timings and such. + * You specialise it for use cases, but a lot of what it needs is generic, + * and it is part of algorithm processing instrumentation. + * In-layer generic usage includes injecting the Graph, hence it is a parameter to the build method. + * Out-layer would be injecting custom dependencies as part of the constructor. + * And the whole build method is bespoke, of course. + * This class is generic in the union type sense, it has fields and accessors for lots of stuff, + * where any given usage probably won't need all of them. + */ +public abstract class ResultBuilder { + // timings + protected long preProcessingMillis; + protected long computeMillis; + protected long postProcessingMillis = -1; // mutate or write timing + + // union type: zero or more of these get populated by your own hooks + protected long nodeCount; + protected long nodePropertiesWritten; + protected long relationshipsWritten; + + public void withPreProcessingMillis(long preProcessingMillis) { + this.preProcessingMillis = preProcessingMillis; + } + + public void withComputeMillis(long computeMillis) { + this.computeMillis = computeMillis; + } + + public void withPostProcessingMillis(long postProcessingMillis) { + this.postProcessingMillis = postProcessingMillis; + } + + public void withNodeCount(long nodeCount) { + this.nodeCount = nodeCount; + } + + public void withNodePropertiesWritten(long nodePropertiesWritten) { + this.nodePropertiesWritten = nodePropertiesWritten; + } + + public void withRelationshipsWritten(long relationshipPropertiesWritten) { + this.relationshipsWritten = relationshipPropertiesWritten; + } + + /** + * You implement this and use as much or as little of the gathered data as is appropriate. + * Plus your own injected dependencies of course. + * + * @param result empty when graph was empty + */ + public abstract RESULT_TO_CALLER build( + Graph graph, + GraphStore graphStore, + Optional result + ); +} diff --git a/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ShortestPathMutateStep.java b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ShortestPathMutateStep.java new file mode 100644 index 00000000000..84bf3dc9a2a --- /dev/null +++ b/applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/ShortestPathMutateStep.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.core.loading.SingleTypeRelationships; +import org.neo4j.gds.core.loading.construction.GraphFactory; +import org.neo4j.gds.paths.dijkstra.PathFindingResult; + +import static org.neo4j.gds.paths.dijkstra.config.ShortestPathDijkstraWriteConfig.TOTAL_COST_KEY; + +class ShortestPathMutateStep implements MutateOrWriteStep { + private final MutateRelationshipConfig configuration; + + ShortestPathMutateStep(MutateRelationshipConfig configuration) { + this.configuration = configuration; + } + + @Override + public void execute( + Graph graph, + GraphStore graphStore, + PathFindingResult result, + ResultBuilder resultBuilder + ) { + var mutateRelationshipType = RelationshipType.of(configuration.mutateRelationshipType()); + + var relationshipsBuilder = GraphFactory + .initRelationshipsBuilder() + .relationshipType(mutateRelationshipType) + .nodes(graph) + .addPropertyConfig(GraphFactory.PropertyConfig.of(TOTAL_COST_KEY)) + .orientation(Orientation.NATURAL) + .build(); + + SingleTypeRelationships relationships; + + result.forEachPath(pathResult -> { + relationshipsBuilder.addFromInternal( + pathResult.sourceNode(), + pathResult.targetNode(), + pathResult.totalCost() + ); + }); + + relationships = relationshipsBuilder.build(); + + // side effect + graphStore.addRelationshipType(relationships); + + // result + resultBuilder.withRelationshipsWritten(relationships.topology().elementCount()); + } +} diff --git a/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplateTest.java b/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplateTest.java new file mode 100644 index 00000000000..02ade098f48 --- /dev/null +++ b/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultAlgorithmProcessingTemplateTest.java @@ -0,0 +1,280 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.DatabaseId; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.GraphName; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.api.User; +import org.neo4j.gds.config.AlgoBaseConfig; +import org.neo4j.gds.config.RelationshipWeightConfig; +import org.neo4j.gds.core.loading.GraphStoreCatalogService; +import org.neo4j.gds.metrics.ExecutionMetric; +import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService; +import org.neo4j.gds.paths.dijkstra.PathFindingResult; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DefaultAlgorithmProcessingTemplateTest { + @Test + void shouldProcessStreamAlgorithm() { + var databaseId = DatabaseId.of("some database"); + var user = new User("some user", false); + var graphStoreCatalogService = mock(GraphStoreCatalogService.class); + var algorithmMetricsService = mock(AlgorithmMetricsService.class); + var algorithmProcessingTemplate = new DefaultAlgorithmProcessingTemplate( + null, + algorithmMetricsService, + graphStoreCatalogService, + MemoryGuard.DISABLED, + databaseId, + user + ); + + var graphName = GraphName.parse("some graph"); + var configuration = new ExampleConfiguration(); + var graph = mock(Graph.class); + var graphStore = mock(GraphStore.class); + when(graphStoreCatalogService.getGraphWithGraphStore( + graphName, + configuration, + Optional.empty(), + user, + databaseId + )).thenReturn(Pair.of(graph, graphStore)); + + // We need it to not be null :shrug: + when(algorithmMetricsService.create("Duckstra")).thenReturn(mock(ExecutionMetric.class)); + + //noinspection unchecked + AlgorithmComputation computation = mock(AlgorithmComputation.class); + var pathFindingResult = mock(PathFindingResult.class); + when(computation.compute(graph)).thenReturn(pathFindingResult); + + var resultBuilder = new ResultBuilder>() { + @Override + public Stream build( + Graph graph, + GraphStore graphStore, + Optional pathFindingResult + ) { + // we skip timings when no side effects requested + assertThat(postProcessingMillis).isEqualTo(-1); + + return Stream.of( + "Huey", + "Dewey", + "Louie" + ); + } + }; + + var resultStream = algorithmProcessingTemplate.processAlgorithm( + graphName, + configuration, + "Duckstra", + null, + computation, + Optional.empty(), + resultBuilder + ); + + assertThat(resultStream).containsExactly( + "Huey", + "Dewey", + "Louie" + ); + } + + /** + * From pathfinding, the contract is that the mutate and write side effects will set relationshipsWritten. + * So that's the example we go with. + */ + @Test + void shouldProcessMutateOrWriteAlgorithm() { + var databaseId = DatabaseId.of("some database"); + var user = new User("some user", false); + var graphStoreCatalogService = mock(GraphStoreCatalogService.class); + var algorithmMetricsService = mock(AlgorithmMetricsService.class); + var algorithmProcessingTemplate = new DefaultAlgorithmProcessingTemplate( + null, + algorithmMetricsService, + graphStoreCatalogService, + MemoryGuard.DISABLED, + databaseId, + user + ); + + var graphName = GraphName.parse("some graph"); + var configuration = new ExampleConfiguration(); + var graph = mock(Graph.class); + var graphStore = mock(GraphStore.class); + when(graphStoreCatalogService.getGraphWithGraphStore( + graphName, + configuration, + Optional.empty(), + user, + databaseId + )).thenReturn(Pair.of(graph, graphStore)); + + var pathFindingResult = mock(PathFindingResult.class); + var resultBuilder = new ResultBuilder() { + @Override + public Long build( + Graph actualGraph, + GraphStore actualGraphStore, + Optional actualResult + ) { + assertThat(actualGraph).isEqualTo(graph); + assertThat(actualGraphStore).isEqualTo(graphStore); + assertThat(actualResult).hasValue(pathFindingResult); + + return relationshipsWritten; + } + }; + + // We need it to not be null :shrug: + when(algorithmMetricsService.create("m || w")).thenReturn(mock(ExecutionMetric.class)); + + //noinspection unchecked + AlgorithmComputation computation = mock(AlgorithmComputation.class); + when(computation.compute(graph)).thenReturn(pathFindingResult); + + var mutateOrWriteStep = new MutateOrWriteStep() { + @Override + public void execute( + Graph graph, + GraphStore graphStore, + PathFindingResult resultFromAlgorithm, + ResultBuilder resultBuilder + ) { + resultBuilder.withRelationshipsWritten(42); + } + }; + + var relationshipsWritten = algorithmProcessingTemplate.processAlgorithm( + graphName, + configuration, + "m || w", + null, + computation, + Optional.of(mutateOrWriteStep), + resultBuilder + ); + + assertThat(relationshipsWritten).isEqualTo(42L); + } + + @Test + void shouldDoTimings() { + var algorithmProcessingTemplate = new DefaultAlgorithmProcessingTemplate( + null, + null, + null, + MemoryGuard.DISABLED, + null, + null + ) { + @Override + Pair graphLoadAndValidationWithTiming( + GraphName graphName, + CONFIGURATION configuration, + ResultBuilder resultBuilder + ) { + resultBuilder.withPreProcessingMillis(23); + return Pair.of(mock(Graph.class), null); + } + + @Override + RESULT_FROM_ALGORITHM computeWithTiming( + String humanReadableAlgorithmName, + AlgorithmComputation algorithmComputation, + ResultBuilder resultBuilder, + Graph graph + ) { + resultBuilder.withComputeMillis(117); + return null; + } + + @Override + void mutateOrWriteWithTiming( + Optional> mutateOrWriteStep, + ResultBuilder resultBuilder, + Graph graph, + GraphStore graphStore, + RESULT_FROM_ALGORITHM resultFromAlgorithm + ) { + resultBuilder.withPostProcessingMillis(87); + } + }; + + var resultBuilder = new ResultBuilder>() { + @Override + public Map build( + Graph graph, + GraphStore graphStore, + Optional unused + ) { + return Map.of( + "preProcessingMillis", preProcessingMillis, + "computeMillis", computeMillis, + "postProcessingMillis", postProcessingMillis + ); + } + }; + + var resultMap = algorithmProcessingTemplate.processAlgorithm( + null, + null, + null, + null, + null, + Optional.of(new MutateOrWriteStep<>() { + @Override + public void execute( + Graph graph, + GraphStore graphStore, + Void unused, + ResultBuilder resultBuilder + ) { + // do nothing, we are just catching timings + } + }), + resultBuilder + ); + + assertThat(resultMap) + .containsOnly( + entry("preProcessingMillis", 23L), + entry("computeMillis", 117L), + entry("postProcessingMillis", 87L) + ); + } +} diff --git a/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuardTest.java b/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuardTest.java new file mode 100644 index 00000000000..34e679966de --- /dev/null +++ b/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/DefaultMemoryGuardTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.GraphDimensions; +import org.neo4j.gds.core.utils.mem.MemoryEstimation; +import org.neo4j.gds.core.utils.mem.MemoryRange; +import org.neo4j.gds.core.utils.mem.MemoryTree; +import org.neo4j.gds.mem.MemoryGauge; + +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * This is a saga of mocks. In my defense, + * it is because the underlying classes aren't written in a tell-don't-ask manner. + * And im not in the mood to change that right this minute. + * So bear with and take it in the best spirit. + */ +class DefaultMemoryGuardTest { + @Test + void shouldAllowExecution() { + var memoryGuard = new DefaultMemoryGuard(null, false, new MemoryGauge(new AtomicLong(42L))); + + var graph = mock(Graph.class); + when(graph.nodeCount()).thenReturn(23L); + when(graph.relationshipCount()).thenReturn(87L); + var memoryEstimation = mock(MemoryEstimation.class); + var memoryTree = mock(MemoryTree.class); + when(memoryEstimation.estimate(GraphDimensions.of(23L, 87L), 7)).thenReturn(memoryTree); + when(memoryTree.memoryUsage()).thenReturn(MemoryRange.of(13, 19)); + + // there is enough memory available + memoryGuard.assertAlgorithmCanRun( + "Duckstra", + new ExampleConfiguration(), + graph, + () -> memoryEstimation + ); + } + + @Test + void shouldGuardExecutionUsingMinimumEstimate() { + var memoryGuard = new DefaultMemoryGuard(null, false, new MemoryGauge(new AtomicLong(42L))); + + var graph = mock(Graph.class); + when(graph.nodeCount()).thenReturn(23L); + when(graph.relationshipCount()).thenReturn(87L); + var memoryEstimation = mock(MemoryEstimation.class); + var memoryTree = mock(MemoryTree.class); + when(memoryEstimation.estimate(GraphDimensions.of(23, 87), 7)).thenReturn(memoryTree); + when(memoryTree.memoryUsage()).thenReturn(MemoryRange.of(117, 243)); + + // uh oh + try { + memoryGuard.assertAlgorithmCanRun( + "Duckstra", + new ExampleConfiguration(), + graph, + () -> memoryEstimation + ); + + fail(); + } catch (IllegalStateException e) { + assertThat(e).hasMessage("Memory required to run Duckstra (117b) exceeds available memory (42b)"); + } + } + + @Test + void shouldGuardExecutionUsingMaximumEstimate() { + var memoryGuard = new DefaultMemoryGuard(null, true, new MemoryGauge(new AtomicLong(42L))); + + var graph = mock(Graph.class); + when(graph.nodeCount()).thenReturn(23L); + when(graph.relationshipCount()).thenReturn(87L); + var memoryEstimation = mock(MemoryEstimation.class); + var memoryTree = mock(MemoryTree.class); + when(memoryEstimation.estimate(GraphDimensions.of(23, 87), 7)).thenReturn(memoryTree); + when(memoryTree.memoryUsage()).thenReturn(MemoryRange.of(117, 243)); + + // uh oh + try { + memoryGuard.assertAlgorithmCanRun( + "Duckstra", + new ExampleConfiguration(), + graph, + () -> memoryEstimation + ); + + fail(); + } catch (IllegalStateException e) { + assertThat(e).hasMessage("Memory required to run Duckstra (243b) exceeds available memory (42b)"); + } + } +} diff --git a/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/ExampleConfiguration.java b/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/ExampleConfiguration.java new file mode 100644 index 00000000000..9107ff9f0d9 --- /dev/null +++ b/applications/algorithms/path-finding/src/test/java/org/neo4j/gds/applications/algorithms/pathfinding/ExampleConfiguration.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.applications.algorithms.pathfinding; + +import org.neo4j.gds.config.AlgoBaseConfig; +import org.neo4j.gds.config.RelationshipWeightConfig; + +import java.util.Optional; + +class ExampleConfiguration implements AlgoBaseConfig, RelationshipWeightConfig { + @Override + public Optional usernameOverride() { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public Optional relationshipWeightProperty() { + return Optional.empty(); + } + + @Override + public int concurrency() { + return 7; + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListener.java b/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListener.java index 9f34d40d1eb..ddfd0bd0a0f 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListener.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListener.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.core.utils.mem; -import org.neo4j.logging.Log; +import org.neo4j.gds.logging.Log; import javax.management.Notification; import javax.management.NotificationFilter; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListenerExtension.java b/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListenerExtension.java index 440469ccf1d..c3ac9970ee4 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListenerExtension.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListenerExtension.java @@ -19,41 +19,24 @@ */ package org.neo4j.gds.core.utils.mem; -import org.neo4j.annotations.service.ServiceProvider; -import org.neo4j.kernel.extension.ExtensionFactory; -import org.neo4j.kernel.extension.context.ExtensionContext; -import org.neo4j.kernel.lifecycle.Lifecycle; -import org.neo4j.logging.internal.LogService; - -import java.lang.management.ManagementFactory; import java.util.concurrent.atomic.AtomicLong; -@ServiceProvider -public final class GcListenerExtension extends ExtensionFactory { +/** + * @deprecated Migrate to using {@link org.neo4j.gds.mem.MemoryGauge} + */ +@Deprecated +public final class GcListenerExtension { // Initialize with max available memory. Everything that is used at this point in time // _could_ be garbage and we want to err on the side of seeing more free heap. // It also has the effect that we allow all operations that theoretically fit into memory // if the extension does never load. - private static final AtomicLong freeMemoryAfterLastGc = new AtomicLong(Runtime.getRuntime().maxMemory()); - - public GcListenerExtension() { - super("gds.heap-control.gc-listener"); - } + private static AtomicLong freeMemoryAfterLastGc = new AtomicLong(Runtime.getRuntime().maxMemory()); public static long freeMemory() { return freeMemoryAfterLastGc.get(); } - @Override - public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies) { - return new GcListenerInstaller( - dependencies.logService(), - ManagementFactory.getGarbageCollectorMXBeans(), - freeMemoryAfterLastGc - ); - } - - interface Dependencies { - LogService logService(); + public static void setMemoryGauge(AtomicLong freeMemoryAfterLastGc) { + GcListenerExtension.freeMemoryAfterLastGc = freeMemoryAfterLastGc; } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/mem/HotSpotGcListener.java b/core/src/main/java/org/neo4j/gds/core/utils/mem/HotSpotGcListener.java index 36dcaf3388a..c0649a6f781 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/mem/HotSpotGcListener.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/mem/HotSpotGcListener.java @@ -19,8 +19,7 @@ */ package org.neo4j.gds.core.utils.mem; -import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.logging.internal.LogService; +import org.neo4j.gds.logging.Log; import javax.management.NotificationBroadcaster; import javax.management.NotificationListener; @@ -34,20 +33,20 @@ import static java.lang.invoke.MethodHandles.filterReturnValue; -final class HotSpotGcListener { +public final class HotSpotGcListener { private static final boolean ENABLED; private static final String GC_NOTIFICATION_NAME; private static final MethodHandle GET_USAGE_AFTER_GC; - static Optional install( - LogService logService, + public static Optional install( + Log log, AtomicLong freeMemory, String[] poolNames, NotificationBroadcaster broadcaster ) { if (ENABLED) { GcListener listener = new GcListener( - Neo4jProxy.getInternalLog(logService, GcListener.class), + log, freeMemory, poolNames, GC_NOTIFICATION_NAME, diff --git a/core/src/test/java/org/neo4j/gds/core/utils/mem/GcListenerTest.java b/core/src/test/java/org/neo4j/gds/core/utils/mem/GcListenerTest.java index fe3e94af6f1..9ed3cb74f60 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/mem/GcListenerTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/mem/GcListenerTest.java @@ -20,7 +20,7 @@ package org.neo4j.gds.core.utils.mem; import org.junit.jupiter.api.Test; -import org.neo4j.logging.NullLog; +import org.neo4j.gds.logging.Log; import javax.management.Notification; import javax.management.NotificationFilter; @@ -50,7 +50,7 @@ class GcListenerTest { @Test void testFilter() { NotificationFilter listener = new GcListener( - NullLog.getInstance(), + Log.noOpLog(), new AtomicLong(-1), new String[0], "some-name-42", @@ -69,7 +69,7 @@ void testListenToEvent() throws OpenDataException { AtomicLong actualFree = new AtomicLong(-1); NotificationListener listener = new GcListener( - NullLog.getInstance(), + Log.noOpLog(), actualFree, new String[]{"global"}, "42", @@ -95,7 +95,7 @@ private MethodHandle testResultHandle(Map usages) { void testCombineMultipleUsages() throws OpenDataException { AtomicLong actualFree = new AtomicLong(-1); NotificationListener listener = new GcListener( - NullLog.getInstance(), + Log.noOpLog(), actualFree, new String[]{ "usedWithMax", diff --git a/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryGauge.java b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryGauge.java new file mode 100644 index 00000000000..24ef8774d32 --- /dev/null +++ b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryGauge.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.mem; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * An oracle we ask for information about system memory. + * We inject an AtomicLong that is populated elsewhere, and read from that. + */ +public class MemoryGauge { + private final AtomicLong availableMemory; + + public MemoryGauge(AtomicLong availableMemory) { + this.availableMemory = availableMemory; + } + + public long availableMemory() { + return availableMemory.get(); + } +} diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProc.java index eff5d4f27a5..92f126e5a45 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProc.java @@ -20,7 +20,6 @@ package org.neo4j.gds.catalog; import org.neo4j.gds.applications.graphstorecatalog.NodePropertiesWriteResult; -import org.neo4j.gds.executor.Preconditions; import org.neo4j.gds.procedures.GraphDataScience; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; @@ -60,8 +59,6 @@ public Stream run( @Name(value = "nodeLabels", defaultValue = "['*']") Object nodeLabels, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { - Preconditions.check(); - facade.deprecatedProcedures().called("gds.graph.writeNodeProperties"); facade.log().warn("Procedure `gds.graph.writeNodeProperties` has been deprecated, please use `gds.graph.nodeProperties.write`."); diff --git a/proc/centrality/src/test/java/org/neo4j/gds/harmonic/DeprecatedTieredHarmonicCentralityWriteProcTest.java b/proc/centrality/src/test/java/org/neo4j/gds/harmonic/DeprecatedTieredHarmonicCentralityWriteProcTest.java index bdd8b107f6a..66484c7e3b8 100644 --- a/proc/centrality/src/test/java/org/neo4j/gds/harmonic/DeprecatedTieredHarmonicCentralityWriteProcTest.java +++ b/proc/centrality/src/test/java/org/neo4j/gds/harmonic/DeprecatedTieredHarmonicCentralityWriteProcTest.java @@ -216,6 +216,7 @@ private GraphDataScience createFacade( null, null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); } diff --git a/proc/centrality/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityWriteProcTest.java b/proc/centrality/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityWriteProcTest.java index ace5f33cb77..24c8dcf1da7 100644 --- a/proc/centrality/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityWriteProcTest.java +++ b/proc/centrality/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityWriteProcTest.java @@ -219,6 +219,7 @@ private GraphDataScience createFacade( null, null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); } diff --git a/proc/community/src/test/java/org/neo4j/gds/k1coloring/K1ColoringStreamProcTest.java b/proc/community/src/test/java/org/neo4j/gds/k1coloring/K1ColoringStreamProcTest.java index 9a4a399c469..63726c6d759 100644 --- a/proc/community/src/test/java/org/neo4j/gds/k1coloring/K1ColoringStreamProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/k1coloring/K1ColoringStreamProcTest.java @@ -236,6 +236,7 @@ void shouldRegisterTaskWithCorrectJobId() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); var someJobId = new JobId(); diff --git a/proc/community/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateProcTest.java b/proc/community/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateProcTest.java index 0fb56b47c4b..f32b7243782 100644 --- a/proc/community/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateProcTest.java @@ -360,6 +360,7 @@ void testWriteBackGraphMutationOnFilteredGraph() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); @@ -494,6 +495,7 @@ void testMutateFailsOnExistingToken() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); @@ -582,6 +584,7 @@ void testRunOnEmptyGraph() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); @@ -674,6 +677,7 @@ private GraphStore runMutation(String graphName, Map config) { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); ProcedureMethodHelper.mutateMethods(procedure) diff --git a/proc/community/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java b/proc/community/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java index e1881bd5da9..fc7c67711b4 100644 --- a/proc/community/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java @@ -609,6 +609,7 @@ private GraphDataScience createFacade() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); } diff --git a/proc/community/src/test/java/org/neo4j/gds/wcc/WccMutateProcTest.java b/proc/community/src/test/java/org/neo4j/gds/wcc/WccMutateProcTest.java index ebbb163ed15..70cd21526fb 100644 --- a/proc/community/src/test/java/org/neo4j/gds/wcc/WccMutateProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/wcc/WccMutateProcTest.java @@ -390,6 +390,7 @@ void testWriteBackGraphMutationOnFilteredGraph() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); @@ -528,6 +529,7 @@ void testMutateFailsOnExistingToken() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); ProcedureMethodHelper.mutateMethods(procedure) @@ -613,6 +615,7 @@ void testRunOnEmptyGraph() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); @@ -706,6 +709,7 @@ private GraphStore runMutation(String graphName, Map additionalC ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); Map config = new HashMap<>(additionalConfig); diff --git a/proc/community/src/test/java/org/neo4j/gds/wcc/WccStatsProcTest.java b/proc/community/src/test/java/org/neo4j/gds/wcc/WccStatsProcTest.java index 109dc64fd65..008c1f40cbf 100644 --- a/proc/community/src/test/java/org/neo4j/gds/wcc/WccStatsProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/wcc/WccStatsProcTest.java @@ -311,6 +311,7 @@ private GraphDataScience createFacade() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); } diff --git a/proc/community/src/test/java/org/neo4j/gds/wcc/WccWriteProcTest.java b/proc/community/src/test/java/org/neo4j/gds/wcc/WccWriteProcTest.java index dbd3776f6a5..da42c1001b2 100644 --- a/proc/community/src/test/java/org/neo4j/gds/wcc/WccWriteProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/wcc/WccWriteProcTest.java @@ -519,6 +519,7 @@ void testRunOnEmptyGraph() { ), null, null, + null, DeprecatedProceduresMetricService.PASSTHROUGH ); diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/PathFindingMutateResultTest.java similarity index 97% rename from proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java rename to proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/PathFindingMutateResultTest.java index f18d8f57a50..a5d37098c63 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/PathFindingMutateResultTest.java @@ -23,7 +23,7 @@ import static org.assertj.core.api.Assertions.assertThatNoException; -class MutateResultTest { +class PathFindingMutateResultTest { @Test void shouldRecordResults() { diff --git a/proc/path-finding/build.gradle b/proc/path-finding/build.gradle index ca574b4d68e..74689b6fd32 100644 --- a/proc/path-finding/build.gradle +++ b/proc/path-finding/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation project(':graph-schema-api') implementation project(':memory-usage') implementation project(':neo4j-api') + implementation project(':opengds-procedure-facade') implementation project(':progress-tracking') implementation project(':string-formatting') implementation project(':termination') diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java index 4f3693a6ccd..4c3ef5fa59b 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java @@ -30,14 +30,15 @@ import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.paths.dijkstra.PathFindingResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.result.AbstractResultBuilder; import static org.neo4j.gds.paths.dijkstra.config.ShortestPathDijkstraWriteConfig.TOTAL_COST_KEY; -public class ShortestPathMutateResultConsumer, CONFIG extends AlgoBaseConfig & MutateRelationshipConfig> extends MutateComputationResultConsumer { +public class ShortestPathMutateResultConsumer, CONFIG extends AlgoBaseConfig & MutateRelationshipConfig> extends MutateComputationResultConsumer { public ShortestPathMutateResultConsumer() { - super((computationResult, executionContext) -> new MutateResult.Builder() + super((computationResult, executionContext) -> new PathFindingMutateResult.Builder() .withPreProcessingMillis(computationResult.preProcessingMillis()) .withComputeMillis(computationResult.computeMillis()) .withConfig(computationResult.config())); diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java index 2ceb24060dc..a0fb2e9b3a0 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java @@ -25,6 +25,7 @@ import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.paths.dijkstra.PathFindingResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import java.util.stream.Stream; @@ -32,10 +33,10 @@ public final class ShortestPathStreamResultConsumer, CONFIG extends AlgoBaseConfig> implements - ComputationResultConsumer> { + ComputationResultConsumer> { @Override - public Stream consume( + public Stream consume( ComputationResult computationResult, ExecutionContext executionContext ) { @@ -48,7 +49,7 @@ public Stream consume( var shouldReturnPath = executionContext.returnColumns() .contains("path") && computationResult.graphStore().capabilities().canWriteToLocalDatabase(); - var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()); + var resultBuilder = new PathFindingStreamResult.Builder(graph, executionContext.nodeLookup()); var resultStream = result.mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamProc.java index c4d161bc573..92cb8a97b97 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamProc.java @@ -21,7 +21,7 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -37,7 +37,7 @@ public class DagLongestPathStreamProc extends BaseProc { @Procedure(value = "gds.dag.longestPath.stream", mode = READ) @Description(LONGEST_PATH_DESCRIPTION) - public Stream stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamSpec.java index fca5b5780c6..52394f8752d 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/dag/longestPath/DagLongestPathStreamSpec.java @@ -28,7 +28,7 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; import org.neo4j.gds.paths.ShortestPathStreamResultConsumer; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.dijkstra.PathFindingResult; import java.util.stream.Stream; @@ -36,7 +36,7 @@ import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.dag.longestPath.stream", description = DagLongestPathStreamProc.LONGEST_PATH_DESCRIPTION, executionMode = STREAM) -public class DagLongestPathStreamSpec implements AlgorithmSpec, DagLongestPathFactory> { +public class DagLongestPathStreamSpec implements AlgorithmSpec, DagLongestPathFactory> { @Override public String name() { @@ -54,7 +54,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateProc.java index cfdee342ebb..1f0944d4fb1 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateProc.java @@ -23,7 +23,7 @@ import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; import org.neo4j.gds.executor.ProcedureExecutorSpec; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.paths.delta.DeltaStepping; import org.neo4j.gds.paths.delta.config.AllShortestPathsDeltaMutateConfig; import org.neo4j.gds.paths.dijkstra.PathFindingResult; @@ -41,7 +41,7 @@ public class AllShortestPathsDeltaMutateProc extends BaseProc { @Procedure(name = "gds.allShortestPaths.delta.mutate", mode = READ) @Description(DeltaStepping.DESCRIPTION) - public Stream stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateSpec.java index 2a3aa34c713..b62790fa7e2 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaMutateSpec.java @@ -24,7 +24,7 @@ import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.paths.ShortestPathMutateResultConsumer; import org.neo4j.gds.paths.delta.DeltaStepping; import org.neo4j.gds.paths.delta.DeltaSteppingFactory; @@ -36,7 +36,7 @@ import static org.neo4j.gds.executor.ExecutionMode.MUTATE_RELATIONSHIP; @GdsCallable(name = "gds.allShortestPaths.delta.mutate", description = DeltaStepping.DESCRIPTION, executionMode = MUTATE_RELATIONSHIP) -public class AllShortestPathsDeltaMutateSpec implements AlgorithmSpec, DeltaSteppingFactory> { +public class AllShortestPathsDeltaMutateSpec implements AlgorithmSpec, DeltaSteppingFactory> { @Override public String name() { @@ -54,7 +54,7 @@ public NewConfigFunction newConfigFunction() } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathMutateResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamProc.java index d7e915ba915..1be971648f8 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamProc.java @@ -23,7 +23,7 @@ import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; import org.neo4j.gds.executor.ProcedureExecutorSpec; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.delta.DeltaStepping; import org.neo4j.gds.paths.delta.config.AllShortestPathsDeltaStreamConfig; import org.neo4j.gds.paths.dijkstra.PathFindingResult; @@ -41,7 +41,7 @@ public class AllShortestPathsDeltaStreamProc extends BaseProc { @Procedure(name = "gds.allShortestPaths.delta.stream", mode = READ) @Description(DeltaStepping.DESCRIPTION) - public Stream stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java index 3e843ba1ed0..c2150eb49ba 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java @@ -25,7 +25,7 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; import org.neo4j.gds.paths.ShortestPathStreamResultConsumer; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.delta.DeltaStepping; import org.neo4j.gds.paths.delta.DeltaSteppingFactory; import org.neo4j.gds.paths.delta.config.AllShortestPathsDeltaStreamConfig; @@ -36,7 +36,7 @@ import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.allShortestPaths.delta.stream", description = DeltaStepping.DESCRIPTION, executionMode = STREAM) -public class AllShortestPathsDeltaStreamSpec implements AlgorithmSpec, DeltaSteppingFactory> { +public class AllShortestPathsDeltaStreamSpec implements AlgorithmSpec, DeltaSteppingFactory> { @Override public String name() { @@ -54,7 +54,7 @@ public NewConfigFunction newConfigFunction() } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateProc.java index 9c3a5420d78..815a00a7f4f 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateProc.java @@ -21,13 +21,14 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; -import org.neo4j.gds.executor.ProcedureExecutor; import org.neo4j.gds.executor.ProcedureExecutorSpec; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.paths.dijkstra.Dijkstra; import org.neo4j.gds.paths.dijkstra.PathFindingResult; import org.neo4j.gds.paths.dijkstra.config.AllShortestPathsDijkstraMutateConfig; +import org.neo4j.gds.procedures.GraphDataScience; import org.neo4j.gds.results.MemoryEstimateResult; +import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -38,21 +39,16 @@ import static org.neo4j.procedure.Mode.READ; public class AllShortestPathsDijkstraMutateProc extends BaseProc { + @Context + public GraphDataScience facade; @Procedure(name = "gds.allShortestPaths.dijkstra.mutate", mode = READ) @Description(Dijkstra.DESCRIPTION_SOURCE_TARGET) - public Stream mutate( + public Stream mutate( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { - var mutateSpec = new AllShortestPathsDijkstraMutateSpec(); - var pipelineSpec = new ProcedureExecutorSpec(); - - return new ProcedureExecutor<>( - mutateSpec, - pipelineSpec, - executionContext() - ).compute(graphName, configuration); + return facade.pathFinding().singleSourceShortestPathDijkstraMutate(graphName, configuration); } @Procedure(name = "gds.allShortestPaths.dijkstra.mutate.estimate", mode = READ) diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateSpec.java index 4c5f279c41d..08c2bf3c2ff 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraMutateSpec.java @@ -24,7 +24,7 @@ import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.paths.ShortestPathMutateResultConsumer; import org.neo4j.gds.paths.dijkstra.Dijkstra; import org.neo4j.gds.paths.dijkstra.DijkstraFactory; @@ -36,7 +36,7 @@ import static org.neo4j.gds.executor.ExecutionMode.MUTATE_RELATIONSHIP; @GdsCallable(name = "gds.allShortestPaths.dijkstra.mutate", description = Dijkstra.DESCRIPTION_SOURCE_TARGET, executionMode = MUTATE_RELATIONSHIP) -public class AllShortestPathsDijkstraMutateSpec implements AlgorithmSpec, DijkstraFactory.AllShortestPathsDijkstraFactory> { +public class AllShortestPathsDijkstraMutateSpec implements AlgorithmSpec, DijkstraFactory.AllShortestPathsDijkstraFactory> { @Override public String name() { @@ -56,7 +56,7 @@ public NewConfigFunction newConfigFunction } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathMutateResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamProc.java index 8ecec374bf2..3a9e0fef8ea 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamProc.java @@ -21,13 +21,14 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; -import org.neo4j.gds.executor.ProcedureExecutor; import org.neo4j.gds.executor.ProcedureExecutorSpec; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.GraphDataScience; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.dijkstra.Dijkstra; import org.neo4j.gds.paths.dijkstra.PathFindingResult; import org.neo4j.gds.paths.dijkstra.config.AllShortestPathsDijkstraStreamConfig; import org.neo4j.gds.results.MemoryEstimateResult; +import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -38,21 +39,16 @@ import static org.neo4j.procedure.Mode.READ; public class AllShortestPathsDijkstraStreamProc extends BaseProc { + @Context + public GraphDataScience facade; @Procedure(name = "gds.allShortestPaths.dijkstra.stream", mode = READ) @Description(Dijkstra.DESCRIPTION_SOURCE_TARGET) - public Stream stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { - var deltaStreamSpec = new AllShortestPathsDijkstraStreamSpec(); - var pipelineSpec = new ProcedureExecutorSpec(); - - return new ProcedureExecutor<>( - deltaStreamSpec, - pipelineSpec, - executionContext() - ).compute(graphName, configuration); + return facade.pathFinding().singleSourceShortestPathDijkstraStream(graphName, configuration); } @Procedure(name = "gds.allShortestPaths.dijkstra.stream.estimate", mode = READ) diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamSpec.java index 50c6beb1d94..415f6c8abd9 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/dijkstra/AllShortestPathsDijkstraStreamSpec.java @@ -25,7 +25,7 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; import org.neo4j.gds.paths.ShortestPathStreamResultConsumer; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.dijkstra.Dijkstra; import org.neo4j.gds.paths.dijkstra.DijkstraFactory; import org.neo4j.gds.paths.dijkstra.PathFindingResult; @@ -36,7 +36,7 @@ import static org.neo4j.gds.executor.ExecutionMode.MUTATE_RELATIONSHIP; @GdsCallable(name = "gds.allShortestPaths.dijkstra.stream", description = Dijkstra.DESCRIPTION_SOURCE_TARGET, executionMode = MUTATE_RELATIONSHIP) -public class AllShortestPathsDijkstraStreamSpec implements AlgorithmSpec, DijkstraFactory.AllShortestPathsDijkstraFactory> { +public class AllShortestPathsDijkstraStreamSpec implements AlgorithmSpec, DijkstraFactory.AllShortestPathsDijkstraFactory> { @Override public String name() { @@ -56,7 +56,7 @@ public NewConfigFunction newConfigFunction } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateProc.java index 215c78c0007..5c8bee3270d 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateProc.java @@ -22,7 +22,7 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -38,7 +38,7 @@ public class ShortestPathAStarMutateProc extends BaseProc { @Procedure(name = "gds.shortestPath.astar.mutate", mode = READ) @Description(ASTAR_DESCRIPTION) - public Stream mutate( + public Stream mutate( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateSpec.java index e4f77ddc6e9..8758598c9ea 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarMutateSpec.java @@ -24,7 +24,7 @@ import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.paths.ShortestPathMutateResultConsumer; import org.neo4j.gds.paths.astar.AStar; import org.neo4j.gds.paths.astar.AStarFactory; @@ -37,7 +37,7 @@ import static org.neo4j.gds.paths.sourcetarget.ShortestPathAStarCompanion.ASTAR_DESCRIPTION; @GdsCallable(name = "gds.shortestPath.astar.mutate", description = ASTAR_DESCRIPTION, executionMode = MUTATE_RELATIONSHIP) -public class ShortestPathAStarMutateSpec implements AlgorithmSpec, AStarFactory> { +public class ShortestPathAStarMutateSpec implements AlgorithmSpec, AStarFactory> { @Override public String name() { @@ -55,7 +55,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathMutateResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProc.java index 884939a37d0..04880d2bb3e 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProc.java @@ -21,9 +21,10 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; -import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.GraphDataScience; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.results.MemoryEstimateResult; +import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -35,17 +36,16 @@ import static org.neo4j.procedure.Mode.READ; public class ShortestPathAStarStreamProc extends BaseProc { + @Context + public GraphDataScience facade; @Procedure(name = "gds.shortestPath.astar.stream", mode = READ) @Description(ASTAR_DESCRIPTION) - public Stream stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { - return new ProcedureExecutor<>( - new ShortestPathAStarStreamSpec(), - executionContext() - ).compute(graphName, configuration); + return facade.pathFinding().singlePairShortestPathAStarStream(graphName, configuration); } @Procedure(name = "gds.shortestPath.astar.stream.estimate", mode = READ) diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamSpec.java index 4cd0aee8af5..a2252b9b2ad 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamSpec.java @@ -25,7 +25,7 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; import org.neo4j.gds.paths.ShortestPathStreamResultConsumer; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.astar.AStar; import org.neo4j.gds.paths.astar.AStarFactory; import org.neo4j.gds.paths.astar.config.ShortestPathAStarStreamConfig; @@ -37,7 +37,7 @@ import static org.neo4j.gds.paths.sourcetarget.ShortestPathAStarCompanion.ASTAR_DESCRIPTION; @GdsCallable(name = "gds.shortestPath.astar.stream", description = ASTAR_DESCRIPTION, executionMode = STREAM) -public class ShortestPathAStarStreamSpec implements AlgorithmSpec, AStarFactory> { +public class ShortestPathAStarStreamSpec implements AlgorithmSpec, AStarFactory> { @Override public String name() { @@ -55,7 +55,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateProc.java index 0b5e1d91136..88261389857 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateProc.java @@ -22,7 +22,7 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -38,7 +38,7 @@ public class ShortestPathDijkstraMutateProc extends BaseProc { @Procedure(name = "gds.shortestPath.dijkstra.mutate", mode = READ) @Description(DIJKSTRA_DESCRIPTION) - public Stream mutate( + public Stream mutate( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateSpec.java index a8c774a2e7a..5b4ea05aa0e 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraMutateSpec.java @@ -24,7 +24,7 @@ import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.paths.ShortestPathMutateResultConsumer; import org.neo4j.gds.paths.dijkstra.Dijkstra; import org.neo4j.gds.paths.dijkstra.DijkstraFactory; @@ -37,7 +37,7 @@ import static org.neo4j.gds.paths.sourcetarget.ShortestPathDijkstraProc.DIJKSTRA_DESCRIPTION; @GdsCallable(name = "gds.shortestPath.dijkstra.mutate", description = DIJKSTRA_DESCRIPTION, executionMode = MUTATE_RELATIONSHIP) -public class ShortestPathDijkstraMutateSpec implements AlgorithmSpec, DijkstraFactory.SourceTargetDijkstraFactory> { +public class ShortestPathDijkstraMutateSpec implements AlgorithmSpec, DijkstraFactory.SourceTargetDijkstraFactory> { @Override public String name() { @@ -57,7 +57,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathMutateResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProc.java index a1b18ed11f8..f5efed1f0ba 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProc.java @@ -21,9 +21,10 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; -import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; +import org.neo4j.gds.procedures.GraphDataScience; import org.neo4j.gds.results.MemoryEstimateResult; +import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -35,17 +36,16 @@ import static org.neo4j.procedure.Mode.READ; public class ShortestPathDijkstraStreamProc extends BaseProc { + @Context + public GraphDataScience facade; @Procedure(name = "gds.shortestPath.dijkstra.stream", mode = READ) @Description(DIJKSTRA_DESCRIPTION) - public Stream stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { - return new ProcedureExecutor<>( - new ShortestPathDijkstraStreamSpec(), - executionContext() - ).compute(graphName, configuration); + return facade.pathFinding().singlePairShortestPathDijkstraStream(graphName, configuration); } @Procedure(name = "gds.shortestPath.dijkstra.stream.estimate", mode = READ) diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamSpec.java index 27785bd11e0..51b46e708e7 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamSpec.java @@ -25,7 +25,7 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; import org.neo4j.gds.paths.ShortestPathStreamResultConsumer; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.dijkstra.Dijkstra; import org.neo4j.gds.paths.dijkstra.DijkstraFactory; import org.neo4j.gds.paths.dijkstra.PathFindingResult; @@ -37,7 +37,7 @@ import static org.neo4j.gds.paths.sourcetarget.ShortestPathDijkstraProc.DIJKSTRA_DESCRIPTION; @GdsCallable(name = "gds.shortestPath.dijkstra.stream", description = DIJKSTRA_DESCRIPTION, executionMode = STREAM) -public class ShortestPathDijkstraStreamSpec implements AlgorithmSpec, DijkstraFactory.SourceTargetDijkstraFactory> { +public class ShortestPathDijkstraStreamSpec implements AlgorithmSpec, DijkstraFactory.SourceTargetDijkstraFactory> { @Override public String name() { @@ -57,7 +57,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateProc.java index fc41ce358ec..5b78d3ce791 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateProc.java @@ -22,7 +22,7 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -37,7 +37,7 @@ public class ShortestPathYensMutateProc extends BaseProc { @Procedure(name = "gds.shortestPath.yens.mutate", mode = READ) @Description(DESCRIPTION) - public Stream mutate( + public Stream mutate( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateSpec.java index 961f11372d9..9681a2bf470 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensMutateSpec.java @@ -24,7 +24,7 @@ import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.paths.ShortestPathMutateResultConsumer; import org.neo4j.gds.paths.dijkstra.PathFindingResult; import org.neo4j.gds.paths.yens.Yens; @@ -37,7 +37,7 @@ import static org.neo4j.gds.paths.sourcetarget.YensConstants.DESCRIPTION; @GdsCallable(name = "gds.shortestPath.yens.mutate", description = DESCRIPTION, executionMode = MUTATE_RELATIONSHIP) -public class ShortestPathYensMutateSpec implements AlgorithmSpec, YensFactory> { +public class ShortestPathYensMutateSpec implements AlgorithmSpec, YensFactory> { @Override public String name() { return "YensMutate"; @@ -54,7 +54,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathMutateResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProc.java index 5f58cebfba4..007cc7c8a4f 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProc.java @@ -22,7 +22,7 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -37,7 +37,7 @@ public class ShortestPathYensStreamProc extends BaseProc { @Procedure(name = "gds.shortestPath.yens.stream", mode = READ) @Description(DESCRIPTION) - public Stream stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamSpec.java index b7ad03505e8..f815b164025 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamSpec.java @@ -25,7 +25,7 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; import org.neo4j.gds.paths.ShortestPathStreamResultConsumer; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.dijkstra.PathFindingResult; import org.neo4j.gds.paths.yens.Yens; import org.neo4j.gds.paths.yens.YensFactory; @@ -36,7 +36,7 @@ import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.shortestPath.yens.stream", description = YensConstants.DESCRIPTION, executionMode = STREAM) -public class ShortestPathYensStreamSpec implements AlgorithmSpec, YensFactory> { +public class ShortestPathYensStreamSpec implements AlgorithmSpec, YensFactory> { @Override public String name() { @@ -54,7 +54,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateComputationResultConsumer.java index 773c8c2bff6..abd45d035cb 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateComputationResultConsumer.java @@ -23,12 +23,12 @@ import org.neo4j.gds.collections.ha.HugeLongArray; import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ExecutionContext; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.result.AbstractResultBuilder; -public class BfsMutateComputationResultConsumer extends MutateComputationResultConsumer { +public class BfsMutateComputationResultConsumer extends MutateComputationResultConsumer { BfsMutateComputationResultConsumer() { - super((computationResult, executionContext) -> new MutateResult.Builder() + super((computationResult, executionContext) -> new PathFindingMutateResult.Builder() .withPreProcessingMillis(computationResult.preProcessingMillis()) .withComputeMillis(computationResult.computeMillis()) .withConfig(computationResult.config())); diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateProc.java index d5e6f2e7d55..44e60f477b8 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateProc.java @@ -22,7 +22,7 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -37,7 +37,7 @@ public class BfsMutateProc extends BaseProc { @Procedure(name = "gds.bfs.mutate", mode = READ) @Description(BfsStreamProc.DESCRIPTION) - public Stream mutate( + public Stream mutate( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateSpec.java index abb9af4f834..0fa340b2df1 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsMutateSpec.java @@ -25,14 +25,14 @@ import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import java.util.stream.Stream; import static org.neo4j.gds.executor.ExecutionMode.MUTATE_RELATIONSHIP; @GdsCallable(name = "gds.bfs.mutate", description = BfsStreamProc.DESCRIPTION, executionMode = MUTATE_RELATIONSHIP) -public class BfsMutateSpec implements AlgorithmSpec, BfsAlgorithmFactory> { +public class BfsMutateSpec implements AlgorithmSpec, BfsAlgorithmFactory> { @Override public String name() { return "gds.bfs.mutate"; @@ -49,7 +49,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new BfsMutateComputationResultConsumer(); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateComputationResultConsumer.java index 9d686134a70..5178ed2ebe0 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateComputationResultConsumer.java @@ -23,12 +23,12 @@ import org.neo4j.gds.collections.ha.HugeLongArray; import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ExecutionContext; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.result.AbstractResultBuilder; -public class DfsMutateComputationResultConsumer extends MutateComputationResultConsumer { +public class DfsMutateComputationResultConsumer extends MutateComputationResultConsumer { DfsMutateComputationResultConsumer() { - super((computationResult, executionContext) -> new MutateResult.Builder() + super((computationResult, executionContext) -> new PathFindingMutateResult.Builder() .withPreProcessingMillis(computationResult.preProcessingMillis()) .withComputeMillis(computationResult.computeMillis()) .withConfig(computationResult.config())); diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateProc.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateProc.java index 1cb73e03639..0fc029cdcd7 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateProc.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateProc.java @@ -22,7 +22,7 @@ import org.neo4j.gds.BaseProc; import org.neo4j.gds.executor.MemoryEstimationExecutor; import org.neo4j.gds.executor.ProcedureExecutor; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -37,7 +37,7 @@ public class DfsMutateProc extends BaseProc { @Procedure(name = "gds.dfs.mutate", mode = READ) @Description(DfsStreamProc.DESCRIPTION) - public Stream mutate( + public Stream mutate( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateSpec.java index f96a298c428..10c88814a48 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsMutateSpec.java @@ -25,14 +25,14 @@ import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import org.neo4j.gds.paths.MutateResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingMutateResult; import java.util.stream.Stream; import static org.neo4j.gds.executor.ExecutionMode.MUTATE_RELATIONSHIP; @GdsCallable(name = "gds.dfs.mutate", description = DfsStreamProc.DESCRIPTION, executionMode = MUTATE_RELATIONSHIP) -public class DfsMutateSpec implements AlgorithmSpec, DfsAlgorithmFactory> { +public class DfsMutateSpec implements AlgorithmSpec, DfsAlgorithmFactory> { @Override public String name() { return "gds.dfs.mutate"; @@ -49,7 +49,7 @@ public NewConfigFunction newConfigFunction() { } @Override - public ComputationResultConsumer> computationResultConsumer() { + public ComputationResultConsumer> computationResultConsumer() { return new DfsMutateComputationResultConsumer(); } } diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProcTest.java index b1faec9677d..df83ec0c352 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathAStarStreamProcTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.Neo4jGraph; import org.neo4j.gds.paths.PathFactory; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.graphdb.RelationshipType; import java.util.Arrays; @@ -137,7 +137,7 @@ void testStream() { tx::getNodeById, ids0, costs0, - RelationshipType.withName(formatWithLocale("PATH_0")), StreamResult.COST_PROPERTY_NAME + RelationshipType.withName(formatWithLocale("PATH_0")), PathFindingStreamResult.COST_PROPERTY_NAME ); var expected = Map.of( "index", 0L, diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProcTest.java index 689e8c70c92..750521e8daa 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathDijkstraStreamProcTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.Neo4jGraph; import org.neo4j.gds.paths.PathFactory; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.graphdb.RelationshipType; import org.neo4j.test.TestDatabaseManagementServiceBuilder; import org.neo4j.test.extension.ExtensionCallback; @@ -125,7 +125,7 @@ void testStream() { tx::getNodeById, ids0, costs0, - RelationshipType.withName(formatWithLocale("PATH_0")), StreamResult.COST_PROPERTY_NAME + RelationshipType.withName(formatWithLocale("PATH_0")), PathFindingStreamResult.COST_PROPERTY_NAME ); var expected = Map.of( "index", 0L, diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java index 5c07a026a6b..83194cc25f1 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java @@ -32,7 +32,7 @@ import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.Neo4jGraph; import org.neo4j.gds.paths.PathFactory; -import org.neo4j.gds.paths.StreamResult; +import org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult; import org.neo4j.gds.paths.yens.config.ShortestPathYensStreamConfig; import org.neo4j.graphdb.RelationshipType; @@ -136,21 +136,21 @@ void testStream() { ids0, costs0, RelationshipType.withName("PATH_0"), - StreamResult.COST_PROPERTY_NAME + PathFindingStreamResult.COST_PROPERTY_NAME ); var path1 = PathFactory.create( tx::getNodeById, ids1, costs1, RelationshipType.withName("PATH_1"), - StreamResult.COST_PROPERTY_NAME + PathFindingStreamResult.COST_PROPERTY_NAME ); var path2 = PathFactory.create( tx::getNodeById, ids2, costs2, RelationshipType.withName("PATH_2"), - StreamResult.COST_PROPERTY_NAME + PathFindingStreamResult.COST_PROPERTY_NAME ); var expected = List.of( Map.of( diff --git a/procedures/extension/build.gradle b/procedures/extension/build.gradle index 24ef25ec571..ee8f454b9c9 100644 --- a/procedures/extension/build.gradle +++ b/procedures/extension/build.gradle @@ -32,6 +32,7 @@ dependencies { implementation project(':neo4j-api') implementation project(':neo4j-kernel-adapter') implementation project(':opengds-procedure-facade') + implementation project(':path-finding-algorithms') implementation project(':proc-catalog') implementation project(':proc-community') implementation project(':proc-sysinfo') diff --git a/procedures/extension/src/main/java/org/neo4j/gds/extension/OpenGraphDataScienceExtension.java b/procedures/extension/src/main/java/org/neo4j/gds/extension/OpenGraphDataScienceExtension.java index 0bc1b68cc9a..53cf43b6632 100644 --- a/procedures/extension/src/main/java/org/neo4j/gds/extension/OpenGraphDataScienceExtension.java +++ b/procedures/extension/src/main/java/org/neo4j/gds/extension/OpenGraphDataScienceExtension.java @@ -21,6 +21,7 @@ import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.configuration.Config; +import org.neo4j.gds.applications.algorithms.pathfinding.AlgorithmProcessingTemplate; import org.neo4j.gds.applications.graphstorecatalog.CatalogBusinessFacade; import org.neo4j.gds.core.write.NativeExportBuildersProvider; import org.neo4j.gds.metrics.MetricsFacade; @@ -35,7 +36,6 @@ import org.neo4j.kernel.extension.ExtensionFactory; import org.neo4j.kernel.extension.context.ExtensionContext; import org.neo4j.kernel.lifecycle.Lifecycle; -import org.neo4j.kernel.lifecycle.LifecycleAdapter; import org.neo4j.logging.internal.LogService; import java.util.Optional; @@ -58,7 +58,8 @@ public Lifecycle newInstance(ExtensionContext extensionContext, Dependencies dep var log = new LogAccessor().getLog(dependencies.logService(), getClass()); var extensionBuilder = ExtensionBuilder.create( - log, dependencies.config(), + log, + dependencies.config(), dependencies.globalProcedures() ); @@ -67,19 +68,23 @@ public Lifecycle newInstance(ExtensionContext extensionContext, Dependencies dep // we have no extra checks to do in OpenGDS Optional> businessFacadeDecorator = Optional.empty(); + Optional> algorithmProcessingTemplateDecorator = Optional.empty(); // metrics are a no-op in OpenGDS var algorithmMetricsService = new AlgorithmMetricsService(new PassthroughExecutionMetricRegistrar()); var projectionMetricsService = new ProjectionMetricsService(new PassthroughExecutionMetricRegistrar()); var metricsFacade = MetricsFacade.PASSTHROUGH_METRICS_FACADE; - extensionBuilder + + // build the extension and return any lifecycles we need managed + return extensionBuilder .withComponent( GraphDataScience.class, () -> extensionBuilder.gdsProvider( exporterBuildersProviderService, businessFacadeDecorator, - metricsFacade + metricsFacade, + algorithmProcessingTemplateDecorator ) ) .withComponent( @@ -87,8 +92,6 @@ public Lifecycle newInstance(ExtensionContext extensionContext, Dependencies dep () -> ctx -> metricsFacade ) .registerExtension(); - - return new LifecycleAdapter(); } public interface Dependencies { diff --git a/procedures/facade/build.gradle b/procedures/facade/build.gradle index e00301dc06a..36c185b5e5d 100644 --- a/procedures/facade/build.gradle +++ b/procedures/facade/build.gradle @@ -34,6 +34,7 @@ dependencies { implementation project(':ml-core') implementation project(':native-projection') implementation project(':neo4j-api') + implementation project(':path-finding-algorithms') implementation project(':proc-common') implementation project(':progress-tracking') implementation project(':string-formatting') diff --git a/procedures/facade/src/main/java/org/neo4j/gds/procedures/GraphDataScience.java b/procedures/facade/src/main/java/org/neo4j/gds/procedures/GraphDataScience.java index e39eeb6ed52..00a797633ae 100644 --- a/procedures/facade/src/main/java/org/neo4j/gds/procedures/GraphDataScience.java +++ b/procedures/facade/src/main/java/org/neo4j/gds/procedures/GraphDataScience.java @@ -24,6 +24,7 @@ import org.neo4j.gds.procedures.catalog.CatalogFacade; import org.neo4j.gds.procedures.centrality.CentralityProcedureFacade; import org.neo4j.gds.procedures.community.CommunityProcedureFacade; +import org.neo4j.gds.procedures.pathfinding.PathFindingProcedureFacade; import org.neo4j.gds.procedures.embeddings.NodeEmbeddingsProcedureFacade; import org.neo4j.gds.procedures.similarity.SimilarityProcedureFacade; @@ -33,6 +34,7 @@ public class GraphDataScience { private final CommunityProcedureFacade communityProcedureFacade; private final CentralityProcedureFacade centralityProcedureFacade; private final SimilarityProcedureFacade similarityProcedureFacade; + private final PathFindingProcedureFacade pathFindingProcedureFacade; private final NodeEmbeddingsProcedureFacade nodeEmbeddingsProcedureFacade; private final DeprecatedProceduresMetricService deprecatedProceduresMetricService; @@ -42,6 +44,7 @@ public GraphDataScience( CommunityProcedureFacade communityProcedureFacade, NodeEmbeddingsProcedureFacade nodeEmbeddingsProcedureFacade, SimilarityProcedureFacade similarityProcedureFacade, + PathFindingProcedureFacade pathFindingProcedureFacade, DeprecatedProceduresMetricService deprecatedProceduresMetricService ) { this.log = log; @@ -49,9 +52,9 @@ public GraphDataScience( this.centralityProcedureFacade = centralityProcedureFacade; this.communityProcedureFacade = communityProcedureFacade; this.similarityProcedureFacade = similarityProcedureFacade; + this.pathFindingProcedureFacade = pathFindingProcedureFacade; this.nodeEmbeddingsProcedureFacade = nodeEmbeddingsProcedureFacade; this.deprecatedProceduresMetricService = deprecatedProceduresMetricService; - } public Log log() { @@ -74,11 +77,14 @@ public SimilarityProcedureFacade similarity() { return similarityProcedureFacade; } + public PathFindingProcedureFacade pathFinding() { + return pathFindingProcedureFacade; + } + public NodeEmbeddingsProcedureFacade nodeEmbeddings() { return nodeEmbeddingsProcedureFacade; } - public DeprecatedProceduresMetricService deprecatedProcedures() { return deprecatedProceduresMetricService; } diff --git a/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/AlgorithmHandle.java b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/AlgorithmHandle.java new file mode 100644 index 00000000000..279915bccf0 --- /dev/null +++ b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/AlgorithmHandle.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.procedures.pathfinding; + +import org.neo4j.gds.api.GraphName; +import org.neo4j.gds.applications.algorithms.pathfinding.ResultBuilder; + +public interface AlgorithmHandle { + RESULT_TO_CALLER compute( + GraphName graphName, + CONFIGURATION configuration, + ResultBuilder resultBuilder + ); +} diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/MutateResult.java b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingMutateResult.java similarity index 82% rename from proc/path-finding/src/main/java/org/neo4j/gds/paths/MutateResult.java rename to procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingMutateResult.java index 42e0cd0bdc0..ca618f75def 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/MutateResult.java +++ b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingMutateResult.java @@ -17,18 +17,17 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.paths; +package org.neo4j.gds.procedures.pathfinding; import org.neo4j.gds.result.AbstractResultBuilder; import org.neo4j.gds.results.StandardMutateResult; import java.util.Map; -@SuppressWarnings("unused") -public final class MutateResult extends StandardMutateResult { +public final class PathFindingMutateResult extends StandardMutateResult { public final long relationshipsWritten; - private MutateResult( + PathFindingMutateResult( long preProcessingMillis, long computeMillis, long mutateMillis, @@ -40,11 +39,10 @@ private MutateResult( this.relationshipsWritten = relationshipsWritten; } - public static class Builder extends AbstractResultBuilder { - + public static class Builder extends AbstractResultBuilder { @Override - public MutateResult build() { - return new MutateResult( + public PathFindingMutateResult build() { + return new PathFindingMutateResult( preProcessingMillis, computeMillis, 0L, diff --git a/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingProcedureFacade.java b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingProcedureFacade.java new file mode 100644 index 00000000000..248d13fd33c --- /dev/null +++ b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingProcedureFacade.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.procedures.pathfinding; + +import org.neo4j.gds.api.CloseableResourceRegistry; +import org.neo4j.gds.api.GraphName; +import org.neo4j.gds.api.NodeLookup; +import org.neo4j.gds.api.ProcedureReturnColumns; +import org.neo4j.gds.applications.algorithms.pathfinding.PathFindingAlgorithmsFacade; +import org.neo4j.gds.config.AlgoBaseConfig; +import org.neo4j.gds.core.CypherMapWrapper; +import org.neo4j.gds.paths.astar.config.ShortestPathAStarStreamConfig; +import org.neo4j.gds.paths.dijkstra.PathFindingResult; +import org.neo4j.gds.paths.dijkstra.config.AllShortestPathsDijkstraMutateConfig; +import org.neo4j.gds.paths.dijkstra.config.AllShortestPathsDijkstraStreamConfig; +import org.neo4j.gds.paths.dijkstra.config.ShortestPathDijkstraStreamConfig; +import org.neo4j.gds.procedures.algorithms.ConfigurationCreator; + +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; + +/** + * This is the top facade on the Neo4j Procedures integration for path finding algorithms. + * The role it plays is, to be newed up with request scoped dependencies, + * and to capture the procedure-specific bits of path finding algorithms calls. + * For example, translating a return column specification into a parameter, a business level concept. + * This is also where we put result rendering. + */ +public class PathFindingProcedureFacade { + // request scoped services + private final CloseableResourceRegistry closeableResourceRegistry; + private final ConfigurationCreator configurationCreator; + private final NodeLookup nodeLookup; + private final ProcedureReturnColumns procedureReturnColumns; + + // delegate + private final PathFindingAlgorithmsFacade facade; + + public PathFindingProcedureFacade( + CloseableResourceRegistry closeableResourceRegistry, + ConfigurationCreator configurationCreator, + NodeLookup nodeLookup, + ProcedureReturnColumns procedureReturnColumns, + PathFindingAlgorithmsFacade facade + ) { + this.closeableResourceRegistry = closeableResourceRegistry; + this.configurationCreator = configurationCreator; + this.nodeLookup = nodeLookup; + this.procedureReturnColumns = procedureReturnColumns; + + this.facade = facade; + } + + public Stream singlePairShortestPathAStarStream( + String graphNameAsString, + Map rawConfiguration + ) { + return runStreamAlgorithm( + graphNameAsString, + rawConfiguration, + ShortestPathAStarStreamConfig::of, + facade::singlePairShortestPathAStarStream + ); + } + + public Stream singlePairShortestPathDijkstraStream( + String graphNameAsString, + Map rawConfiguration + ) { + return runStreamAlgorithm( + graphNameAsString, + rawConfiguration, + ShortestPathDijkstraStreamConfig::of, + facade::singlePairShortestPathDijkstraStream + ); + } + + public Stream singleSourceShortestPathDijkstraMutate( + String graphNameAsString, + Map rawConfiguration + ) { + return Stream.of( + runMutateAlgorithm( + graphNameAsString, + rawConfiguration, + AllShortestPathsDijkstraMutateConfig::of, + facade::singleSourceShortestPathDijkstraMutate + ) + ); + } + + public Stream singleSourceShortestPathDijkstraStream( + String graphNameAsString, + Map rawConfiguration + ) { + return runStreamAlgorithm( + graphNameAsString, + rawConfiguration, + AllShortestPathsDijkstraStreamConfig::of, + facade::singleSourceShortestPathDijkstraStream + ); + } + + /** + * This is kind of a template method: Dijkstra and A* both use the same code structure. + * It is quite banal: + *

    + *
  1. configuration parsing + *
  2. parameter marshalling + *
  3. delegating to down stream layer to call the thing we are actually interested in + *
  4. handle resource closure + *
+ */ + private Stream runStreamAlgorithm( + String graphNameAsString, + Map rawConfiguration, + Function configurationSupplier, + AlgorithmHandle> algorithm + ) { + var graphName = GraphName.parse(graphNameAsString); + var configuration = configurationCreator.createConfigurationForStream(rawConfiguration, configurationSupplier); + var resultBuilder = new PathFindingResultBuilderForStreamMode( + nodeLookup, + procedureReturnColumns.contains("path") + ); + + var resultStream = algorithm.compute(graphName, configuration, resultBuilder); + + // we need to do this for stream mode + closeableResourceRegistry.register(resultStream); + + return resultStream; + } + + /** + * Contract this with {@link #runStreamAlgorithm}: + *
    + *
  • Same input handling + *
  • Pick a different result marshaller - this is the big responsibility in this layer + *
  • Delegate to compute + *
  • No stream closing hook + *
+ * + * So very much the same, but I didn't fancy trying to extract reuse today, for readability's sake + */ + private PathFindingMutateResult runMutateAlgorithm( + String graphNameAsString, + Map rawConfiguration, + Function configurationSupplier, + AlgorithmHandle algorithm + ) { + var graphName = GraphName.parse(graphNameAsString); + var configuration = configurationCreator.createConfiguration(rawConfiguration, configurationSupplier); + var resultBuilder = new PathFindingResultBuilderForMutateMode(configuration); + + return algorithm.compute(graphName, configuration, resultBuilder); + } +} diff --git a/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForMutateMode.java b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForMutateMode.java new file mode 100644 index 00000000000..f66bff83578 --- /dev/null +++ b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForMutateMode.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.procedures.pathfinding; + +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.applications.algorithms.pathfinding.ResultBuilder; +import org.neo4j.gds.config.ToMapConvertible; +import org.neo4j.gds.paths.dijkstra.PathFindingResult; + +import java.util.Optional; + +class PathFindingResultBuilderForMutateMode extends ResultBuilder { + private final ToMapConvertible configuration; + + PathFindingResultBuilderForMutateMode(ToMapConvertible configuration) { + this.configuration = configuration; + } + + @Override + public PathFindingMutateResult build( + Graph graph, + GraphStore graphStore, + Optional pathFindingResult + ) { + return new PathFindingMutateResult( + preProcessingMillis, + computeMillis, + 0, // yeah, I don't understand it either :shrug: + postProcessingMillis, + relationshipsWritten, + configuration.toMap() + ); + } +} diff --git a/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForStreamMode.java b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForStreamMode.java new file mode 100644 index 00000000000..4cb63980eca --- /dev/null +++ b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingResultBuilderForStreamMode.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.procedures.pathfinding; + +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.api.NodeLookup; +import org.neo4j.gds.applications.algorithms.pathfinding.ResultBuilder; +import org.neo4j.gds.paths.dijkstra.PathFindingResult; +import org.neo4j.graphdb.Path; +import org.neo4j.graphdb.RelationshipType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.neo4j.gds.paths.PathFactory.create; +import static org.neo4j.gds.procedures.pathfinding.PathFindingStreamResult.COST_PROPERTY_NAME; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public class PathFindingResultBuilderForStreamMode extends ResultBuilder> { + private final NodeLookup nodeLookup; + private final boolean pathRequested; + + PathFindingResultBuilderForStreamMode(NodeLookup nodeLookup, boolean pathRequested) { + this.nodeLookup = nodeLookup; + this.pathRequested = pathRequested; + } + + @Override + public Stream build( + Graph graph, + GraphStore graphStore, + Optional result + ) { + if (result.isEmpty()) return Stream.of(); + + // this is us handling the case of generated graphs and such + var createCypherPaths = pathRequested && graphStore.capabilities().canWriteToLocalDatabase(); + + return result.get().mapPaths(pathResult -> { + var nodeIds = pathResult.nodeIds(); + var costs = pathResult.costs(); + var pathIndex = pathResult.index(); + + var relationshipType = RelationshipType.withName(formatWithLocale("PATH_%d", pathIndex)); + + // convert internal ids to Neo ids + for (int i = 0; i < nodeIds.length; i++) { + nodeIds[i] = graph.toOriginalNodeId(nodeIds[i]); + } + + Path path = null; + if (createCypherPaths) { + path = create( + nodeLookup, + nodeIds, + costs, + relationshipType, + COST_PROPERTY_NAME + ); + } + + return new PathFindingStreamResult( + pathIndex, + graph.toOriginalNodeId(pathResult.sourceNode()), + graph.toOriginalNodeId(pathResult.targetNode()), + pathResult.totalCost(), + // 😿 + Arrays.stream(nodeIds) + .boxed() + .collect(Collectors.toCollection(() -> new ArrayList<>(nodeIds.length))), + Arrays.stream(costs) + .boxed() + .collect(Collectors.toCollection(() -> new ArrayList<>(costs.length))), + path + ); + }); + } +} diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/StreamResult.java b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingStreamResult.java similarity index 91% rename from proc/path-finding/src/main/java/org/neo4j/gds/paths/StreamResult.java rename to procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingStreamResult.java index b7e65f8c542..3927df59450 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/StreamResult.java +++ b/procedures/facade/src/main/java/org/neo4j/gds/procedures/pathfinding/PathFindingStreamResult.java @@ -17,11 +17,12 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.paths; +package org.neo4j.gds.procedures.pathfinding; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.NodeLookup; +import org.neo4j.gds.paths.PathResult; import org.neo4j.graphdb.Path; import org.neo4j.graphdb.RelationshipType; @@ -33,8 +34,7 @@ import static org.neo4j.gds.paths.PathFactory.create; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -@SuppressWarnings("unused") -public final class StreamResult { +public final class PathFindingStreamResult { public static final String COST_PROPERTY_NAME = "cost"; @@ -52,7 +52,7 @@ public final class StreamResult { public Path path; - private StreamResult( + PathFindingStreamResult( long index, long sourceNode, long targetNode, @@ -79,7 +79,7 @@ public Builder(IdMap idMap, NodeLookup nodeLookup) { this.nodeLookup = nodeLookup; } - public StreamResult build(PathResult pathResult, boolean createCypherPath) { + public PathFindingStreamResult build(PathResult pathResult, boolean createCypherPath) { var nodeIds = pathResult.nodeIds(); var costs = pathResult.costs(); var pathIndex = pathResult.index(); @@ -104,7 +104,7 @@ public StreamResult build(PathResult pathResult, boolean createCypherPath) { } - return new StreamResult( + return new PathFindingStreamResult( pathIndex, idMap.toOriginalNodeId(pathResult.sourceNode()), idMap.toOriginalNodeId(pathResult.targetNode()), diff --git a/procedures/integration/build.gradle b/procedures/integration/build.gradle index bad20621aa1..417ceea6acf 100644 --- a/procedures/integration/build.gradle +++ b/procedures/integration/build.gradle @@ -22,11 +22,13 @@ dependencies { implementation project(':legacy-cypher-projection') implementation project(':logging') implementation project(':memory-estimation') + implementation project(':memory-usage') implementation project(':metrics-api') implementation project(':native-projection') implementation project(':neo4j-api') implementation project(':neo4j-settings') implementation project(':opengds-procedure-facade') + implementation project(':path-finding-algorithms') implementation project(':proc-common') implementation project(':proc-community') implementation project(':progress-tracking') diff --git a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmFacadeProviderFactory.java b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmFacadeProviderFactory.java index fca0ba3b8e1..b4a607c707c 100644 --- a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmFacadeProviderFactory.java +++ b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmFacadeProviderFactory.java @@ -20,6 +20,8 @@ package org.neo4j.gds.procedures.integration; import org.neo4j.gds.ProcedureCallContextReturnColumns; +import org.neo4j.gds.TransactionCloseableResourceRegistry; +import org.neo4j.gds.TransactionNodeLookup; import org.neo4j.gds.algorithms.AlgorithmMemoryValidationService; import org.neo4j.gds.algorithms.RequestScopedDependencies; import org.neo4j.gds.algorithms.estimation.AlgorithmEstimator; @@ -28,9 +30,13 @@ import org.neo4j.gds.algorithms.similarity.MutateRelationshipService; import org.neo4j.gds.algorithms.similarity.WriteRelationshipService; import org.neo4j.gds.algorithms.writeservices.WriteNodePropertyService; +import org.neo4j.gds.applications.algorithms.pathfinding.AlgorithmProcessingTemplate; +import org.neo4j.gds.applications.algorithms.pathfinding.DefaultAlgorithmProcessingTemplate; +import org.neo4j.gds.applications.algorithms.pathfinding.DefaultMemoryGuard; import org.neo4j.gds.core.loading.GraphStoreCatalogService; import org.neo4j.gds.core.write.ExporterContext; import org.neo4j.gds.logging.Log; +import org.neo4j.gds.mem.MemoryGauge; import org.neo4j.gds.memest.DatabaseGraphStoreEstimationService; import org.neo4j.gds.memest.FictitiousGraphStoreEstimationService; import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService; @@ -45,6 +51,9 @@ import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.kernel.api.procedure.Context; +import java.util.Optional; +import java.util.function.Function; + class AlgorithmFacadeProviderFactory { // dull utilities @@ -54,6 +63,7 @@ class AlgorithmFacadeProviderFactory { private final Log log; private final ConfigurationParser configurationParser; private final GraphStoreCatalogService graphStoreCatalogService; + private final MemoryGauge memoryGauge; private final boolean useMaxMemoryEstimation; // Request scoped state and services @@ -66,6 +76,7 @@ class AlgorithmFacadeProviderFactory { private final TerminationFlagService terminationFlagService; private final UserAccessor userAccessor; private final UserLogServices userLogServices; + private final Optional> algorithmProcessingTemplateDecorator; //algorithm facade parameters @@ -73,6 +84,7 @@ class AlgorithmFacadeProviderFactory { Log log, ConfigurationParser configurationParser, GraphStoreCatalogService graphStoreCatalogService, + MemoryGauge memoryGauge, boolean useMaxMemoryEstimation, AlgorithmMetaDataSetterService algorithmMetaDataSetterService, AlgorithmMetricsService algorithmMetricsService, @@ -81,11 +93,14 @@ class AlgorithmFacadeProviderFactory { KernelTransactionAccessor kernelTransactionAccessor, TaskRegistryFactoryService taskRegistryFactoryService, TerminationFlagService terminationFlagService, - UserAccessor userAccessor, UserLogServices userLogServices + UserAccessor userAccessor, + UserLogServices userLogServices, + Optional> algorithmProcessingTemplateDecorator ) { this.log = log; this.configurationParser = configurationParser; this.graphStoreCatalogService = graphStoreCatalogService; + this.memoryGauge = memoryGauge; this.useMaxMemoryEstimation = useMaxMemoryEstimation; this.algorithmMetaDataSetterService = algorithmMetaDataSetterService; @@ -97,6 +112,7 @@ class AlgorithmFacadeProviderFactory { this.terminationFlagService = terminationFlagService; this.userLogServices = userLogServices; this.userAccessor = userAccessor; + this.algorithmProcessingTemplateDecorator = algorithmProcessingTemplateDecorator; } @@ -108,7 +124,9 @@ AlgorithmProcedureFacadeProvider createAlgorithmFacadeProvider(Context context) // GDS services derived from Procedure Context var algorithmMetaDataSetter = algorithmMetaDataSetterService.getAlgorithmMetaDataSetter(kernelTransaction); var algorithmMemoryValidationService = new AlgorithmMemoryValidationService(log, useMaxMemoryEstimation); + var closeableResourceRegistry = new TransactionCloseableResourceRegistry(kernelTransaction); var databaseId = databaseIdAccessor.getDatabaseId(context.graphDatabaseAPI()); + var nodeLookup = new TransactionNodeLookup(kernelTransaction); var returnColumns = new ProcedureCallContextReturnColumns(context.procedureCallContext()); var terminationFlag = terminationFlagService.createTerminationFlag(kernelTransaction); var user = userAccessor.getUser(context.securityContext()); @@ -170,17 +188,38 @@ AlgorithmProcedureFacadeProvider createAlgorithmFacadeProvider(Context context) var mutateNodePropertyService = new MutateNodePropertyService(log); var mutateRelationshipService = new MutateRelationshipService(log); var configurationCreator = new ConfigurationCreator(configurationParser, algorithmMetaDataSetter, user); + + var memoryGuard = new DefaultMemoryGuard(log, useMaxMemoryEstimation, memoryGauge); + + AlgorithmProcessingTemplate algorithmProcessingTemplate = new DefaultAlgorithmProcessingTemplate( + log, + algorithmMetricsService, + graphStoreCatalogService, + memoryGuard, + databaseId, + user + ); + + if (this.algorithmProcessingTemplateDecorator.isPresent()) + algorithmProcessingTemplate = this.algorithmProcessingTemplateDecorator.get().apply(algorithmProcessingTemplate); + // procedure facade return new AlgorithmProcedureFacadeProvider( + log, + closeableResourceRegistry, configurationCreator, + nodeLookup, returnColumns, mutateNodePropertyService, writeNodePropertyService, mutateRelationshipService, writeRelationshipService, algorithmRunner, - algorithmEstimator + algorithmEstimator, + algorithmProcessingTemplate, + taskRegistryFactory, + terminationFlag, + userLogRegistryFactory ); } - } diff --git a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmProcedureFacadeProvider.java b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmProcedureFacadeProvider.java index 92cf9f3de89..ba73a42f72d 100644 --- a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmProcedureFacadeProvider.java +++ b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/AlgorithmProcedureFacadeProvider.java @@ -49,15 +49,30 @@ import org.neo4j.gds.algorithms.similarity.SimilarityAlgorithmsWriteBusinessFacade; import org.neo4j.gds.algorithms.similarity.WriteRelationshipService; import org.neo4j.gds.algorithms.writeservices.WriteNodePropertyService; +import org.neo4j.gds.api.CloseableResourceRegistry; +import org.neo4j.gds.api.NodeLookup; +import org.neo4j.gds.applications.algorithms.pathfinding.AlgorithmProcessingTemplate; +import org.neo4j.gds.applications.algorithms.pathfinding.PathFindingAlgorithms; +import org.neo4j.gds.applications.algorithms.pathfinding.PathFindingAlgorithmsFacade; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory; +import org.neo4j.gds.logging.Log; import org.neo4j.gds.procedures.algorithms.ConfigurationCreator; import org.neo4j.gds.procedures.centrality.CentralityProcedureFacade; import org.neo4j.gds.procedures.community.CommunityProcedureFacade; import org.neo4j.gds.procedures.embeddings.NodeEmbeddingsProcedureFacade; +import org.neo4j.gds.procedures.pathfinding.PathFindingProcedureFacade; import org.neo4j.gds.procedures.similarity.SimilarityProcedureFacade; +import org.neo4j.gds.termination.TerminationFlag; class AlgorithmProcedureFacadeProvider { + // Global scoped dependencies + private final Log log; + // Request scoped parameters + private final CloseableResourceRegistry closeableResourceRegistry; private final ConfigurationCreator configurationCreator; + private final NodeLookup nodeLookup; private final ProcedureCallContextReturnColumns returnColumns; private final MutateNodePropertyService mutateNodePropertyService; private final WriteNodePropertyService writeNodePropertyService; @@ -65,18 +80,32 @@ class AlgorithmProcedureFacadeProvider { private final WriteRelationshipService writeRelationshipService; private final AlgorithmEstimator algorithmEstimator; private final AlgorithmRunner algorithmRunner; + private final AlgorithmProcessingTemplate algorithmProcessingTemplate; + private final TaskRegistryFactory taskRegistryFactory; + private final TerminationFlag terminationFlag; + private final UserLogRegistryFactory userLogRegistryFactory; AlgorithmProcedureFacadeProvider( + Log log, + CloseableResourceRegistry closeableResourceRegistry, ConfigurationCreator configurationCreator, + NodeLookup nodeLookup, ProcedureCallContextReturnColumns returnColumns, MutateNodePropertyService mutateNodePropertyService, WriteNodePropertyService writeNodePropertyService, MutateRelationshipService mutateRelationshipService, WriteRelationshipService writeRelationshipService, AlgorithmRunner algorithmRunner, - AlgorithmEstimator algorithmEstimator + AlgorithmEstimator algorithmEstimator, + AlgorithmProcessingTemplate algorithmProcessingTemplate, + TaskRegistryFactory taskRegistryFactory, + TerminationFlag terminationFlag, + UserLogRegistryFactory userLogRegistryFactory ) { + this.log = log; + this.closeableResourceRegistry = closeableResourceRegistry; this.configurationCreator = configurationCreator; + this.nodeLookup = nodeLookup; this.returnColumns = returnColumns; this.mutateNodePropertyService = mutateNodePropertyService; this.writeNodePropertyService = writeNodePropertyService; @@ -85,6 +114,10 @@ class AlgorithmProcedureFacadeProvider { this.algorithmRunner = algorithmRunner; this.algorithmEstimator = algorithmEstimator; + this.algorithmProcessingTemplate = algorithmProcessingTemplate; + this.taskRegistryFactory = taskRegistryFactory; + this.terminationFlag = terminationFlag; + this.userLogRegistryFactory = userLogRegistryFactory; } CentralityProcedureFacade createCentralityProcedureFacade() { @@ -176,6 +209,28 @@ SimilarityProcedureFacade createSimilarityProcedureFacade() { } + PathFindingProcedureFacade createPathFindingProcedureFacade() { + var pathFindingAlgorithms = new PathFindingAlgorithms( + log, + taskRegistryFactory, + terminationFlag, + userLogRegistryFactory + ); + + var pathFindingAlgorithmsFacade = new PathFindingAlgorithmsFacade( + algorithmProcessingTemplate, + pathFindingAlgorithms + ); + + return new PathFindingProcedureFacade( + closeableResourceRegistry, + configurationCreator, + nodeLookup, + returnColumns, + pathFindingAlgorithmsFacade + ); + } + NodeEmbeddingsProcedureFacade createNodeEmbeddingsProcedureFacade() { // algorithms facade var nodeEmbeddingsAlgorithmsFacade = new NodeEmbeddingsAlgorithmsFacade(algorithmRunner); @@ -205,7 +260,5 @@ NodeEmbeddingsProcedureFacade createNodeEmbeddingsProcedureFacade() { streamBusinessFacade, writeBusinessFacade ); - } - } diff --git a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/ExtensionBuilder.java b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/ExtensionBuilder.java index eed2f8400b4..b44d89c017d 100644 --- a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/ExtensionBuilder.java +++ b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/ExtensionBuilder.java @@ -20,16 +20,19 @@ package org.neo4j.gds.procedures.integration; import org.neo4j.function.ThrowingFunction; +import org.neo4j.gds.applications.algorithms.pathfinding.AlgorithmProcessingTemplate; import org.neo4j.gds.applications.graphstorecatalog.CatalogBusinessFacade; import org.neo4j.gds.configuration.DefaultsConfiguration; import org.neo4j.gds.configuration.LimitsConfiguration; import org.neo4j.gds.core.loading.GraphStoreCatalogService; +import org.neo4j.gds.core.utils.mem.GcListenerExtension; import org.neo4j.gds.core.utils.progress.ProgressFeatureSettings; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; import org.neo4j.gds.core.utils.progress.TaskStore; import org.neo4j.gds.core.utils.progress.TaskStoreService; import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory; import org.neo4j.gds.logging.Log; +import org.neo4j.gds.mem.MemoryGauge; import org.neo4j.gds.metrics.MetricsFacade; import org.neo4j.gds.metrics.algorithms.AlgorithmMetricsService; import org.neo4j.gds.metrics.projections.ProjectionMetricsService; @@ -46,10 +49,14 @@ import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.kernel.api.procedure.Context; import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.lifecycle.LifeSupport; +import org.neo4j.kernel.lifecycle.Lifecycle; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Collection; import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.function.Supplier; @@ -74,32 +81,38 @@ public final class ExtensionBuilder { private final GlobalProcedures globalProcedures; // GDS state and services - private final TaskStoreService taskStoreService; - private final TaskRegistryFactoryService taskRegistryFactoryService; - private final UserLogServices userLogServices; private final ConfigurationParser configurationParser; private final GraphStoreCatalogService graphStoreCatalogService; + private final MemoryGauge memoryGauge; + private final TaskStoreService taskStoreService; + private final TaskRegistryFactoryService taskRegistryFactoryService; private final boolean useMaxMemoryEstimation; + private final UserLogServices userLogServices; + private final Lifecycle[] lifecycles; private ExtensionBuilder( Log log, GlobalProcedures globalProcedures, + ConfigurationParser configurationParser, + GraphStoreCatalogService graphStoreCatalogService, + MemoryGauge memoryGauge, TaskStoreService taskStoreService, TaskRegistryFactoryService taskRegistryFactoryService, + boolean useMaxMemoryEstimation, UserLogServices userLogServices, - ConfigurationParser configurationParser, - GraphStoreCatalogService graphStoreCatalogService, - boolean useMaxMemoryEstimation + Lifecycle... lifecycles ) { this.log = log; this.globalProcedures = globalProcedures; - this.taskStoreService = taskStoreService; - this.taskRegistryFactoryService = taskRegistryFactoryService; - this.userLogServices = userLogServices; this.configurationParser = configurationParser; this.graphStoreCatalogService = graphStoreCatalogService; + this.memoryGauge = memoryGauge; + this.taskStoreService = taskStoreService; + this.taskRegistryFactoryService = taskRegistryFactoryService; this.useMaxMemoryEstimation = useMaxMemoryEstimation; + this.userLogServices = userLogServices; + this.lifecycles = lifecycles; } /** @@ -129,15 +142,35 @@ public static ExtensionBuilder create( // Speaking of state, defaults and limits is a big shared thing also (or, will be) var configurationParser = new ConfigurationParser(DefaultsConfiguration.Instance, LimitsConfiguration.Instance); + // Memory gauge integrates with the JVM + // First, it is state held in an AtomicLong + // Initialize with max available memory. Everything that is used at this point in time + // _could_ be garbage and we want to err on the side of seeing more free heap. + // It also has the effect that we allow all operations that theoretically fit into memory + // if the extension does never load. + AtomicLong freeMemoryAfterLastGc = new AtomicLong(Runtime.getRuntime().maxMemory()); + // We make it available in a neat service + var memoryGauge = new MemoryGauge(freeMemoryAfterLastGc); + // in the short term, until we eradicate old usages, we also install the shared state in its old place + GcListenerExtension.setMemoryGauge(freeMemoryAfterLastGc); + // State is populated from a GC listener + Lifecycle gcListener = new GcListenerInstaller( + log, + ManagementFactory.getGarbageCollectorMXBeans(), + freeMemoryAfterLastGc + ); + return new ExtensionBuilder( log, globalProcedures, + configurationParser, + graphStoreCatalogService, + memoryGauge, taskStoreService, taskRegistryFactoryService, + useMaxMemoryEstimation, userLogServices, - configurationParser, - graphStoreCatalogService, - useMaxMemoryEstimation + gcListener ); } @@ -161,7 +194,7 @@ public ExtensionBuilder withComponent( /** * The finalisation of the builder registers components with Neo4j */ - public void registerExtension() { + public Lifecycle registerExtension() { // Process the list of components to register registrations.forEach(Runnable::run); @@ -193,6 +226,13 @@ public void registerExtension() { true ); log.info("User Log Registry registered."); + + // Lastly we sort out the lifecycles we need to have managed + var lifeSupport = new LifeSupport(); + for (Lifecycle lifecycle : lifecycles) { + lifeSupport.add(lifecycle); + } + return lifeSupport; } /** @@ -206,7 +246,8 @@ public void registerExtension() { public ThrowingFunction gdsProvider( ExporterBuildersProviderService exporterBuildersProviderService, Optional> businessFacadeDecorator, - MetricsFacade metricsFacade + MetricsFacade metricsFacade, + Optional> algorithmProcessingTemplateDecorator ) { var catalogFacadeProvider = createCatalogFacadeProvider( exporterBuildersProviderService, @@ -216,15 +257,14 @@ public ThrowingFunction gdsProvid var algorithmFacadeService = createAlgorithmService( metricsFacade.algorithmMetrics(), - exporterBuildersProviderService + exporterBuildersProviderService, + algorithmProcessingTemplateDecorator ); - return new GraphDataScienceProvider( log, catalogFacadeProvider, algorithmFacadeService, - metricsFacade.deprecatedProcedures() ); } @@ -254,12 +294,14 @@ private CatalogFacadeProvider createCatalogFacadeProvider( private AlgorithmFacadeProviderFactory createAlgorithmService( AlgorithmMetricsService algorithmMetricsService, - ExporterBuildersProviderService exporterBuildersProviderService + ExporterBuildersProviderService exporterBuildersProviderService, + Optional> algorithmProcessingTemplateDecorator ) { return new AlgorithmFacadeProviderFactory( log, configurationParser, graphStoreCatalogService, + memoryGauge, useMaxMemoryEstimation, algorithmMetaDataSetterService, algorithmMetricsService, @@ -269,8 +311,8 @@ private AlgorithmFacadeProviderFactory createAlgorithmService( taskRegistryFactoryService, terminationFlagService, userAccessor, - userLogServices + userLogServices, + algorithmProcessingTemplateDecorator ); } - } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListenerInstaller.java b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/GcListenerInstaller.java similarity index 92% rename from core/src/main/java/org/neo4j/gds/core/utils/mem/GcListenerInstaller.java rename to procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/GcListenerInstaller.java index 70e50f129ed..dd91c80fab7 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/mem/GcListenerInstaller.java +++ b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/GcListenerInstaller.java @@ -17,10 +17,11 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.core.utils.mem; +package org.neo4j.gds.procedures.integration; +import org.neo4j.gds.core.utils.mem.HotSpotGcListener; +import org.neo4j.gds.logging.Log; import org.neo4j.kernel.lifecycle.LifecycleAdapter; -import org.neo4j.logging.internal.LogService; import javax.management.ListenerNotFoundException; import javax.management.NotificationBroadcaster; @@ -36,17 +37,17 @@ import java.util.concurrent.atomic.AtomicLong; final class GcListenerInstaller extends LifecycleAdapter { - private final LogService logService; + private final Log log; private final List gcBeans; private final AtomicLong freeMemory; private final Map registeredListeners; GcListenerInstaller( - LogService logService, + Log log, Collection gcBeans, AtomicLong freeMemory ) { - this.logService = logService; + this.log = log; // make defensive copy this.gcBeans = new ArrayList<>(gcBeans); this.freeMemory = freeMemory; @@ -69,7 +70,7 @@ private void installGcListener(MemoryManagerMXBean gcBean) { if (gcBean instanceof NotificationBroadcaster) { NotificationBroadcaster broadcaster = (NotificationBroadcaster) gcBean; Optional listener = HotSpotGcListener.install( - this.logService, + log, this.freeMemory, gcBean.getMemoryPoolNames(), broadcaster diff --git a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/GraphDataScienceProvider.java b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/GraphDataScienceProvider.java index ba05ab482f4..1a2afc69a52 100644 --- a/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/GraphDataScienceProvider.java +++ b/procedures/integration/src/main/java/org/neo4j/gds/procedures/integration/GraphDataScienceProvider.java @@ -53,8 +53,10 @@ public GraphDataScience apply(Context context) throws ProcedureException { var catalogFacade = catalogFacadeProvider.createCatalogFacade(context); var algorithmFacadeProvider = algorithmFacadeService.createAlgorithmFacadeProvider(context); + var centralityProcedureFacade = algorithmFacadeProvider.createCentralityProcedureFacade(); var communityProcedureFacade = algorithmFacadeProvider.createCommunityProcedureFacade(); + var pathFindingProcedureFacade = algorithmFacadeProvider.createPathFindingProcedureFacade(); var similarityProcedureFacade = algorithmFacadeProvider.createSimilarityProcedureFacade(); var nodeEmbeddingsProcedureFacade = algorithmFacadeProvider.createNodeEmbeddingsProcedureFacade(); @@ -65,6 +67,7 @@ public GraphDataScience apply(Context context) throws ProcedureException { communityProcedureFacade, nodeEmbeddingsProcedureFacade, similarityProcedureFacade, + pathFindingProcedureFacade, deprecatedProceduresMetricService ); } diff --git a/settings.gradle b/settings.gradle index 78b72b02f05..b2ba1d5495b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -89,6 +89,9 @@ project(':executor').projectDir = file('executor') include('graph-store-catalog-applications') project(':graph-store-catalog-applications').projectDir = file('applications/graph-store-catalog') +include('path-finding-algorithms') +project(':path-finding-algorithms').projectDir = file('applications/algorithms/path-finding') + include 'procedure-collector' project(':procedure-collector').projectDir = file('procedure-collector')