From 34afc3dfb07cb4327b079c4662e4581b044f0b0a Mon Sep 17 00:00:00 2001 From: Sam Carlberg Date: Wed, 17 Apr 2019 11:41:22 -0400 Subject: [PATCH] Add CUDA acceleration (#933) * Add CUDA support * Add Socket.flagChanged() No more socket.setValue(socket.getValue()) hacks * Allow CUDA versions to be optionally specified with -Pcuda Or -PWITH_CUDA * Add special CUDA socket Socket is disabled when CUDA is unavailable * Exit when CUDA runtime is required but not available * Make normalize operation CUDA-accelerated Slight speedup versus CPU * Add CUDA acceleration to basic CV operations * Clean up flagChanged in OutputSocketImpl * Append 'cuda' to versions of GRIP with CUDA acceleration * Add CUDA classes to dedicated cuda module This lets us create CudaDetectors etc. before loading OpenCV in the core module Move logger setup to its own class, since the core module may not load before app exits * Make SafeShutdown take an ExitCode enum instead of raw int * Use JNI loading to locate CUDA install * Use WriteProperties task to save CUDA runtime properties --- core/core.gradle.kts | 28 +- .../edu/wpi/grip/core/GripCoreModule.java | 74 +--- .../edu/wpi/grip/core/GripCudaModule.java | 54 +++ .../main/java/edu/wpi/grip/core/Loggers.java | 70 ++++ .../src/main/java/edu/wpi/grip/core/Main.java | 19 +- .../java/edu/wpi/grip/core/MatWrapper.java | 382 ++++++++++++++++++ .../wpi/grip/core/cuda/AccelerationMode.java | 13 + .../grip/core/cuda/CudaAccelerationMode.java | 8 + .../edu/wpi/grip/core/cuda/CudaDetector.java | 13 + .../edu/wpi/grip/core/cuda/CudaVerifier.java | 71 ++++ .../grip/core/cuda/LoadingCudaDetector.java | 36 ++ .../grip/core/cuda/NullAccelerationMode.java | 8 + .../wpi/grip/core/cuda/NullCudaDetector.java | 8 + .../core/events/UnexpectedThrowableEvent.java | 4 +- .../grip/core/operations/CVOperations.java | 319 ++++++++++----- .../grip/core/operations/CudaOperation.java | 85 ++++ .../operations/composite/BlobsReport.java | 11 +- .../operations/composite/BlurOperation.java | 121 +++++- .../composite/CannyEdgeOperation.java | 93 +++++ .../composite/CascadeClassifierOperation.java | 12 +- .../composite/DesaturateOperation.java | 40 +- .../composite/DistanceTransformOperation.java | 15 +- .../composite/FindBlobsOperation.java | 9 +- .../composite/FindContoursOperation.java | 8 +- .../composite/FindLinesOperation.java | 11 +- .../composite/HSLThresholdOperation.java | 15 +- .../composite/HSVThresholdOperation.java | 15 +- .../operations/composite/LinesReport.java | 10 +- .../operations/composite/MaskOperation.java | 21 +- .../composite/NormalizeOperation.java | 45 ++- .../composite/PublishVideoOperation.java | 13 +- .../composite/RGBThresholdOperation.java | 15 +- .../operations/composite/RectsReport.java | 11 +- .../operations/composite/ResizeOperation.java | 17 +- .../composite/SaveImageOperation.java | 23 +- .../operations/composite/ThresholdMoving.java | 15 +- .../composite/WatershedOperation.java | 7 +- .../network/networktables/NTManager.java | 1 + .../network/ros/JavaToMessageConverter.java | 2 +- .../operations/opencv/MatFieldAccessor.java | 19 +- .../core/operations/opencv/MinMaxLoc.java | 28 +- ...FiveSourceOneDestinationCudaOperation.java | 79 ++++ .../FiveSourceOneDestinationOperation.java | 2 +- ...FourSourceOneDestinationCudaOperation.java | 74 ++++ .../FourSourceOneDestinationOperation.java | 2 +- .../OneSourceOneDestinationCudaOperation.java | 55 +++ .../OneSourceOneDestinationOperation.java | 2 +- ...evenSourceOneDestinationCudaOperation.java | 91 +++++ .../SevenSourceOneDestinationOperation.java | 2 +- .../SixSourceOneDestinationCudaOperation.java | 84 ++++ .../SixSourceOneDestinationOperation.java | 2 +- .../operations/templated/TemplateFactory.java | 132 +++++- ...hreeSourceOneDestinationCudaOperation.java | 69 ++++ .../ThreeSourceOneDestinationOperation.java | 2 +- .../TwoSourceOneDestinationCudaOperation.java | 64 +++ .../TwoSourceOneDestinationOperation.java | 2 +- .../edu/wpi/grip/core/sockets/CudaSocket.java | 62 +++ .../wpi/grip/core/sockets/InputSocket.java | 2 + .../grip/core/sockets/InputSocketImpl.java | 10 +- .../edu/wpi/grip/core/sockets/Socket.java | 9 + .../edu/wpi/grip/core/sockets/SocketHint.java | 4 +- .../wpi/grip/core/sockets/SocketHints.java | 19 +- .../edu/wpi/grip/core/sockets/SocketImpl.java | 6 + .../wpi/grip/core/sources/CameraSource.java | 10 +- .../edu/wpi/grip/core/sources/HttpSource.java | 16 +- .../grip/core/sources/ImageFileSource.java | 9 +- .../core/sources/MultiImageFileSource.java | 16 +- .../grip/core/sources/VideoFileSource.java | 7 +- .../edu/wpi/grip/core/util/MetaInfReader.java | 10 +- .../edu/wpi/grip/core/util/SafeShutdown.java | 42 +- .../java/edu/wpi/grip/core/AddOperation.java | 17 +- .../edu/wpi/grip/core/CoreSanityTest.java | 3 +- .../edu/wpi/grip/core/PipelineRunnerTest.java | 1 + .../wpi/grip/core/cuda/CudaVerifierTest.java | 51 +++ .../grip/core/operations/OperationsUtil.java | 7 +- .../operations/opencv/AddOperationTest.java | 13 +- .../grip/core/serialization/ProjectTest.java | 18 +- .../core/sockets/MockInputSocketFactory.java | 4 +- .../grip/core/sources/CameraSourceTest.java | 2 +- .../wpi/grip/core/sources/HttpSourceTest.java | 4 +- .../core/sources/ImageFileSourceTest.java | 8 +- .../sources/MultiImageFileSourceTest.java | 10 +- .../wpi/grip/core/util/MetaInfReaderTest.java | 2 +- .../wpi/grip/core/util/SafeShutdownTest.java | 2 +- .../java/edu/wpi/grip/util/ImageWithData.java | 5 + .../edu/wpi/grip/ui/DeployController.java | 2 + ui/src/main/java/edu/wpi/grip/ui/Main.java | 34 +- .../edu/wpi/grip/ui/MainWindowController.java | 2 +- .../ui/codegeneration/data/TPipeline.java | 5 + .../ui/components/PreviousNextButtons.java | 1 + .../ui/pipeline/OutputSocketController.java | 6 +- .../grip/ui/pipeline/PipelineController.java | 2 + .../input/CheckboxInputSocketController.java | 17 +- .../ui/preview/BlobsSocketPreviewView.java | 2 +- .../ui/preview/ImageBasedPreviewView.java | 4 +- .../ui/preview/ImageSocketPreviewView.java | 11 +- .../ui/preview/LinesSocketPreviewView.java | 2 +- .../preview/RectangleSocketPreviewView.java | 2 +- .../ui/preview/SocketPreviewViewFactory.java | 8 +- .../edu/wpi/grip/ui/util/ImageConverter.java | 5 + .../ui/codegeneration/tools/HelperTools.java | 4 +- .../preview/ImageSocketPreviewViewTest.java | 6 +- ui/ui.gradle.kts | 6 + 103 files changed, 2413 insertions(+), 506 deletions(-) create mode 100644 core/src/main/java/edu/wpi/grip/core/GripCudaModule.java create mode 100644 core/src/main/java/edu/wpi/grip/core/Loggers.java create mode 100644 core/src/main/java/edu/wpi/grip/core/MatWrapper.java create mode 100644 core/src/main/java/edu/wpi/grip/core/cuda/AccelerationMode.java create mode 100644 core/src/main/java/edu/wpi/grip/core/cuda/CudaAccelerationMode.java create mode 100644 core/src/main/java/edu/wpi/grip/core/cuda/CudaDetector.java create mode 100644 core/src/main/java/edu/wpi/grip/core/cuda/CudaVerifier.java create mode 100644 core/src/main/java/edu/wpi/grip/core/cuda/LoadingCudaDetector.java create mode 100644 core/src/main/java/edu/wpi/grip/core/cuda/NullAccelerationMode.java create mode 100644 core/src/main/java/edu/wpi/grip/core/cuda/NullCudaDetector.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationCudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationCudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationCudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationCudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationCudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationCudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationCudaOperation.java create mode 100644 core/src/main/java/edu/wpi/grip/core/sockets/CudaSocket.java create mode 100644 core/src/test/java/edu/wpi/grip/core/cuda/CudaVerifierTest.java diff --git a/core/core.gradle.kts b/core/core.gradle.kts index 3092b71326..a1e1ce870d 100644 --- a/core/core.gradle.kts +++ b/core/core.gradle.kts @@ -1,4 +1,6 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import com.google.common.io.Files +import java.nio.charset.StandardCharsets plugins { id("java") @@ -15,15 +17,21 @@ application { val os = osdetector.classifier.replace("osx", "macosx").replace("x86_32", "x86") val arch = osdetector.arch.replace("x86_64", "x64") +val withCuda = project.hasProperty("cuda") || project.hasProperty("WITH_CUDA") + +if (withCuda) { + version = "$version-cuda" +} + dependencies { api(project(":annotation")) annotationProcessor(project(":annotation")) api(group = "com.google.code.findbugs", name = "jsr305", version = "3.0.1") - api(group = "org.bytedeco", name = "javacv", version = "1.1") - api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.0.0-1.1") - api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.0.0-1.1", classifier = os) + api(group = "org.bytedeco", name = "javacv", version = "1.3") + api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.4.3-1.4.3") + api(group = "org.bytedeco.javacpp-presets", name = "opencv", version = "3.4.3-1.4.3", classifier = if (withCuda) "$os-gpu" else os) api(group = "org.bytedeco.javacpp-presets", name = "videoinput", version = "0.200-1.1", classifier = os) - api(group = "org.bytedeco.javacpp-presets", name = "ffmpeg", version = "0.200-1.1", classifier = os) + api(group = "org.bytedeco.javacpp-presets", name = "ffmpeg", version = "0.200-1.3", classifier = os) api(group = "org.python", name = "jython", version = "2.7.0") api(group = "com.thoughtworks.xstream", name = "xstream", version = "1.4.10") api(group = "org.apache.commons", name = "commons-lang3", version = "3.5") @@ -59,6 +67,18 @@ tasks.withType().configureEach { } } +val writeCudaPropertiesTask = tasks.register("writeCudaProperties") { + description = "Generates a file to let the GRIP runtime know if its using CUDA-accelerated OpenCV." + outputFile = buildDir.resolve("resources/main/edu/wpi/grip/core/CUDA.properties") + comment = "Information about CUDA requirements for the GRIP runtime" + property("edu.wpi.grip.cuda.enabled", withCuda) + property("edu.wpi.grip.cuda.version", "10.0") // opencv-presets 3.4.3-1.4.3 is compiled with CUDA 10.0 +} + +tasks.withType().configureEach { + dependsOn(writeCudaPropertiesTask) +} + tasks.withType().configureEach { /* The icudt54b directory in Jython takes up 9 megabytes and doesn"t seem to do anything useful. */ exclude("org/python/icu/impl/data/icudt54b/") diff --git a/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java b/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java index d1b6e274f6..572e450b51 100644 --- a/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java +++ b/core/src/main/java/edu/wpi/grip/core/GripCoreModule.java @@ -1,5 +1,10 @@ package edu.wpi.grip.core; +import edu.wpi.grip.core.cuda.AccelerationMode; +import edu.wpi.grip.core.cuda.CudaDetector; +import edu.wpi.grip.core.cuda.CudaVerifier; +import edu.wpi.grip.core.cuda.NullAccelerationMode; +import edu.wpi.grip.core.cuda.NullCudaDetector; import edu.wpi.grip.core.events.EventLogger; import edu.wpi.grip.core.events.UnexpectedThrowableEvent; import edu.wpi.grip.core.metrics.BenchmarkRunner; @@ -23,22 +28,18 @@ import com.google.common.eventbus.EventBus; import com.google.common.eventbus.SubscriberExceptionContext; import com.google.inject.AbstractModule; +import com.google.inject.Scopes; import com.google.inject.TypeLiteral; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.matcher.Matchers; +import com.google.inject.name.Names; import com.google.inject.spi.InjectionListener; import com.google.inject.spi.TypeEncounter; import com.google.inject.spi.TypeListener; -import java.io.IOException; -import java.util.logging.FileHandler; -import java.util.logging.Handler; +import java.util.Properties; import java.util.logging.Level; -import java.util.logging.LogManager; -import java.util.logging.LogRecord; import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; -import java.util.logging.StreamHandler; import javax.annotation.Nullable; @@ -53,55 +54,6 @@ public class GripCoreModule extends AbstractModule { private static final Logger logger = Logger.getLogger(GripCoreModule.class.getName()); - // This is in a static initialization block so that we don't create a ton of - // log files when running tests - static { - //Set up the global level logger. This handles IO for all loggers. - final Logger globalLogger = LogManager.getLogManager().getLogger(""); - - try { - // Remove the default handlers that stream to System.err - for (Handler handler : globalLogger.getHandlers()) { - globalLogger.removeHandler(handler); - } - - GripFileManager.GRIP_DIRECTORY.mkdirs(); - final Handler fileHandler - = new FileHandler(GripFileManager.GRIP_DIRECTORY.getPath() + "/GRIP.log"); - - //Set level to handler and logger - fileHandler.setLevel(Level.INFO); - globalLogger.setLevel(Level.INFO); - - // We need to stream to System.out instead of System.err - final StreamHandler sh = new StreamHandler(System.out, new SimpleFormatter()) { - - @Override - public synchronized void publish(final LogRecord record) { - super.publish(record); - // For some reason this doesn't happen automatically. - // This will ensure we get all of the logs printed to the console immediately - // when running on a remote device. - flush(); - } - }; - sh.setLevel(Level.CONFIG); - - globalLogger.addHandler(sh); // Add stream handler - - globalLogger.addHandler(fileHandler); //Add the handler to the global logger - - fileHandler.setFormatter(new SimpleFormatter()); //log in text, not xml - - globalLogger.config("Configuration done."); //Log that we are done setting up the logger - globalLogger.config("GRIP Version: " + edu.wpi.grip.core.Main.class.getPackage() - .getImplementationVersion()); - - } catch (IOException exception) { //Something happened setting up file IO - throw new IllegalStateException("Failed to configure the Logger", exception); - } - } - /* * This class should not be used in tests. Use GRIPCoreTestModule for tests. */ @@ -138,6 +90,16 @@ public void hear(TypeLiteral type, TypeEncounter encounter) { bind(ConnectionValidator.class).to(Pipeline.class); bind(Source.SourceFactory.class).to(Source.SourceFactoryImpl.class); + // Bind CUDA-specific stuff to default values + // These will be overridden by the GripCudaModule at app runtime, but this lets + // automated tests assume CPU-only operation modes + bind(CudaDetector.class).to(NullCudaDetector.class); + bind(AccelerationMode.class).to(NullAccelerationMode.class); + bind(CudaVerifier.class).in(Scopes.SINGLETON); + bind(Properties.class) + .annotatedWith(Names.named("cudaProperties")) + .toInstance(new Properties()); + bind(InputSocket.Factory.class).to(InputSocketImpl.FactoryImpl.class); bind(OutputSocket.Factory.class).to(OutputSocketImpl.FactoryImpl.class); install(new FactoryModuleBuilder() diff --git a/core/src/main/java/edu/wpi/grip/core/GripCudaModule.java b/core/src/main/java/edu/wpi/grip/core/GripCudaModule.java new file mode 100644 index 0000000000..1b12362e18 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/GripCudaModule.java @@ -0,0 +1,54 @@ +package edu.wpi.grip.core; + +import edu.wpi.grip.core.cuda.AccelerationMode; +import edu.wpi.grip.core.cuda.CudaAccelerationMode; +import edu.wpi.grip.core.cuda.CudaDetector; +import edu.wpi.grip.core.cuda.CudaVerifier; +import edu.wpi.grip.core.cuda.LoadingCudaDetector; +import edu.wpi.grip.core.cuda.NullAccelerationMode; + +import com.google.inject.AbstractModule; +import com.google.inject.Scopes; +import com.google.inject.name.Names; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class GripCudaModule extends AbstractModule { + + private static final Logger logger = Logger.getLogger(GripCudaModule.class.getName()); + private static final String CUDA_ENABLED_KEY = "edu.wpi.grip.cuda.enabled"; + + @Override + protected void configure() { + bind(CudaDetector.class).to(LoadingCudaDetector.class); + + Properties cudaProperties = getCudaProperties(); + bind(Properties.class) + .annotatedWith(Names.named("cudaProperties")) + .toInstance(cudaProperties); + + if (Boolean.valueOf(cudaProperties.getProperty(CUDA_ENABLED_KEY, "false"))) { + bind(AccelerationMode.class).to(CudaAccelerationMode.class); + } else { + bind(AccelerationMode.class).to(NullAccelerationMode.class); + } + + bind(CudaVerifier.class).in(Scopes.SINGLETON); + } + + private Properties getCudaProperties() { + try (InputStream resourceAsStream = getClass().getResourceAsStream("CUDA.properties")) { + Properties cudaProps = new Properties(); + cudaProps.load(resourceAsStream); + return cudaProps; + } catch (IOException e) { + logger.log(Level.WARNING, "Could not read CUDA properties", e); + return new Properties(); + } + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/Loggers.java b/core/src/main/java/edu/wpi/grip/core/Loggers.java new file mode 100644 index 0000000000..31418e0443 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/Loggers.java @@ -0,0 +1,70 @@ +package edu.wpi.grip.core; + +import java.io.IOException; +import java.util.logging.FileHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; +import java.util.logging.StreamHandler; + +public final class Loggers { + + private Loggers() { + throw new UnsupportedOperationException("This is a utility class!"); + } + + /** + * Sets up loggers to print to stdout and to ~/GRIP/GRIP.log. This should only be called once + * in the application lifecycle, at startup. + */ + public static void setupLoggers() { + // Set up the global level logger. This handles IO for all loggers. + final Logger globalLogger = LogManager.getLogManager().getLogger(""); + + try { + // Remove the default handlers that stream to System.err + for (Handler handler : globalLogger.getHandlers()) { + globalLogger.removeHandler(handler); + } + + GripFileManager.GRIP_DIRECTORY.mkdirs(); + final Handler fileHandler + = new FileHandler(GripFileManager.GRIP_DIRECTORY.getPath() + "/GRIP.log"); + + //Set level to handler and logger + fileHandler.setLevel(Level.INFO); + globalLogger.setLevel(Level.INFO); + + // We need to stream to System.out instead of System.err + final StreamHandler sh = new StreamHandler(System.out, new SimpleFormatter()) { + + @Override + public synchronized void publish(final LogRecord record) { + super.publish(record); + // For some reason this doesn't happen automatically. + // This will ensure we get all of the logs printed to the console immediately + // when running on a remote device. + flush(); + } + }; + sh.setLevel(Level.CONFIG); + + globalLogger.addHandler(sh); // Add stream handler + + globalLogger.addHandler(fileHandler); //Add the handler to the global logger + + fileHandler.setFormatter(new SimpleFormatter()); //log in text, not xml + + globalLogger.config("Configuration done."); //Log that we are done setting up the logger + globalLogger.config("GRIP Version: " + edu.wpi.grip.core.Main.class.getPackage() + .getImplementationVersion()); + + } catch (IOException exception) { //Something happened setting up file IO + throw new IllegalStateException("Failed to configure the Logger", exception); + } + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/Main.java b/core/src/main/java/edu/wpi/grip/core/Main.java index d58ca3e9b5..e06def2808 100644 --- a/core/src/main/java/edu/wpi/grip/core/Main.java +++ b/core/src/main/java/edu/wpi/grip/core/Main.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core; +import edu.wpi.grip.core.cuda.CudaVerifier; import edu.wpi.grip.core.events.ExceptionClearedEvent; import edu.wpi.grip.core.events.ExceptionEvent; import edu.wpi.grip.core.exception.GripServerException; @@ -55,8 +56,20 @@ public class Main { @SuppressWarnings("JavadocMethod") public static void main(String[] args) throws IOException, InterruptedException { new CoreCommandLineHelper().parse(args); // Check for help or version before doing anything else - final Injector injector = Guice.createInjector(Modules.override(new GripCoreModule(), - new GripFileModule(), new GripSourcesHardwareModule()).with(new GripNetworkModule())); + Loggers.setupLoggers(); + + // Verify CUDA before using the core module, since that will cause OpenCV to be loaded, + // which will crash the app if we use CUDA and it's not available + GripCudaModule cudaModule = new GripCudaModule(); + CudaVerifier cudaVerifier = Guice.createInjector(cudaModule).getInstance(CudaVerifier.class); + cudaVerifier.verifyCuda(); + + final Injector injector = Guice.createInjector( + Modules.override( + new GripCoreModule(), + new GripFileModule(), + new GripSourcesHardwareModule() + ).with(new GripNetworkModule(), cudaModule)); injector.getInstance(Main.class).start(args); } @@ -81,7 +94,7 @@ public void start(String[] args) throws IOException, InterruptedException { gripServer.start(); } catch (GripServerException e) { logger.log(Level.SEVERE, "The HTTP server could not be started", e); - SafeShutdown.exit(1); + SafeShutdown.exit(SafeShutdown.ExitCode.HTTP_SERVER_COULD_NOT_START); } if (pipelineRunner.state() == Service.State.NEW) { diff --git a/core/src/main/java/edu/wpi/grip/core/MatWrapper.java b/core/src/main/java/edu/wpi/grip/core/MatWrapper.java new file mode 100644 index 0000000000..5636a68662 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/MatWrapper.java @@ -0,0 +1,382 @@ +package edu.wpi.grip.core; + +import org.bytedeco.javacpp.opencv_core; +import org.bytedeco.javacpp.opencv_core.GpuMat; +import org.bytedeco.javacpp.opencv_core.Mat; +import org.bytedeco.javacpp.opencv_core.Size; + +import java.util.Objects; +import java.util.function.Function; + +import static org.bytedeco.javacpp.opencv_core.CV_16S; +import static org.bytedeco.javacpp.opencv_core.CV_16U; +import static org.bytedeco.javacpp.opencv_core.CV_32F; +import static org.bytedeco.javacpp.opencv_core.CV_32S; +import static org.bytedeco.javacpp.opencv_core.CV_64F; +import static org.bytedeco.javacpp.opencv_core.CV_8S; +import static org.bytedeco.javacpp.opencv_core.CV_8U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_16S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_16U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_1U; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_32F; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_32S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_64F; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_8S; +import static org.bytedeco.javacpp.opencv_core.IPL_DEPTH_8U; + +/** + * Wraps a GPU mat and a CPU mat and allows device memory and host memory + * to be used semi-transparently. A wrapper may change between wrapping an image in host memory or + * an image in GPU memory. A wrapper is used to minimize copies between host and device memory, + * which may take longer than the time savings of using a CUDA-accelerated operation. + * + *

Data is lazily copied between host and device memory when needed. Wrappers that + * are only accessed from CPU operations will never have their data stored in device memory. + * Accessing a wrapper from CUDA land with {@link #getGpu()} will first copy the existing data from + * host memory to device, if the wrapper was most recently accessed from CPU code (from either + * {@link #getCpu()} or {@link #rawCpu()}). This behavior also applies in the reverse, for wrappers + * accessed from CPU land when they have been most recently used from CUDA code. + */ +@SuppressWarnings("PMD.GodClass") +public final class MatWrapper { + + private final Mat cpuMat; + private final GpuMat gpuMat; + + /** + * Flags whether or not the wrapped data is in host memory. Data may still be available in device + * memory, but the value in host memory will be the working version. + */ + private boolean isCpu = true; + + /** + * Flags whether or not the wrapped value has been modified since the most recent read. This + * is used so that the value is only copied between host and device when it's needed. + */ + private boolean changed = false; + + /** + * Creates an empty wrapper. Both mats are empty and the wrapper is treated as a CPU mat. + */ + public static MatWrapper emptyWrapper() { + return new MatWrapper(new Mat(), new GpuMat()); + } + + /** + * Creates a wrapper around an existing mat in host memory. + */ + public static MatWrapper wrap(Mat cpuMat) { + return new MatWrapper(cpuMat, new GpuMat()); + } + + /** + * Creates a wrapper around an existing mat in GPU memory. + */ + public static MatWrapper wrap(GpuMat gpuMat) { + return new MatWrapper(new Mat(), gpuMat); + } + + /** + * Creates a wrapper around existing mats in host and device memory. + * + * @param cpuMat the mat in host memory to use + * @param gpuMat the mat in device memory to use + */ + public static MatWrapper using(Mat cpuMat, GpuMat gpuMat) { + return new MatWrapper(cpuMat, gpuMat); + } + + private MatWrapper(Mat cpuMat, GpuMat gpuMat) { + this.cpuMat = Objects.requireNonNull(cpuMat, "cpuMat"); + this.gpuMat = Objects.requireNonNull(gpuMat, "gpuMat"); + } + + /** + * Checks if this is a wrapper around a CPU mat. + */ + public boolean isCpu() { + return isCpu; + } + + /** + * Checks if this is a wrapper around a GPU mat. + */ + public boolean isGpu() { + return !isCpu; + } + + /** + * Gets the raw CPU mat. This should only be used when this mat is used as a {@code dst} parameter + * to an OpenCV function. If you want to get the current value as a mat in host memory, use + * {@link #getCpu()}. + */ + public Mat rawCpu() { + // Assume the mat is about to be modified as a `dst` parameter to an OpenCV function + // running on the CPU + isCpu = true; + changed = true; + return cpuMat; + } + + /** + * Gets the raw GPU mat. This should only be used when this mat is used as a {@code dst} parameter + * to an OpenCV function. If you want to get the current value as a mat in GPU memory, use + * {@link #getGpu()}. + */ + public GpuMat rawGpu() { + // Assume the mat is about to be modified as a `dst` parameter to an OpenCV function + // running on a CUDA device + isCpu = false; + changed = true; + return gpuMat; + } + + /** + * Gets this mat as a mat in host memory. If this is {@link #isGpu() backed by GPU memory}, the + * device memory will be copied into the CPU mat before being returned. This copy only happens + * after {@link #set(GpuMat) set(GpuMat)} is called, and only once between successive calls; + * invocations of this method after the first copy will not perform another. + */ + public Mat getCpu() { + if (changed && !isCpu) { + gpuMat.download(cpuMat); + changed = false; + } + return cpuMat; + } + + /** + * Gets this mat as a mat in GPU memory. If this is {@link #isCpu() backed by host memory}, the + * host memory will be copied into the GPU mat before being returned. This copy only happens + * after {@link #set(Mat) set(Mat)} is called, and only once between successive calls; + * invocations of this method after the first copy will not perform another. + */ + public GpuMat getGpu() { + if (changed && isCpu) { + gpuMat.upload(cpuMat); + changed = false; + } + return gpuMat; + } + + /** + * Sets this as being backed by an image in host memory. The data in the given mat will be copied + * into the internal CPU mat, but not the GPU mat until {@link #getGpu()} is called. This avoids + * unnecessary memory copies between GPU and host memory. + */ + public void set(Mat mat) { + mat.copyTo(cpuMat); + isCpu = true; + changed = true; + } + + /** + * Sets this as being backed by an image in GPU memory. The data in the given mat will be copied + * into the internal GPU mat, but not the mat residing in host memory until {@link #getCpu()} is + * called. This avoids unnecessary memory copies between GPU and host memory. + */ + public void set(GpuMat mat) { + gpuMat.put(mat); + isCpu = false; + changed = true; + } + + /** + * Sets this as being backed by the given wrapper. This wrapper will be functionally equivalent + * to the one given. + */ + public void set(MatWrapper wrapper) { + if (wrapper.isCpu()) { + set(wrapper.cpuMat); + } else { + set(wrapper.gpuMat); + } + } + + /** + * Copies the data of this wrapper to a mat in host memory. + */ + public void copyTo(Mat mat) { + if (isCpu) { + cpuMat.copyTo(mat); + } else { + gpuMat.download(mat); + } + } + + /** + * Copies the data of this wrapper to a mat in GPU memory. + */ + public void copyTo(GpuMat mat) { + if (isCpu) { + mat.upload(cpuMat); + } else { + mat.put(gpuMat); + } + } + + /** + * Copies the data of this wrapper into another. Equivalent to {@code wrapper.set(this)} + */ + public void copyTo(MatWrapper wrapper) { + wrapper.set(this); + } + + /** + * Extracts a property shared by both Mats and GpuMats. Unfortunately, they don't share a common + * API, so we have to do something like this. + *

+ * Example use: + * 


+   * Size size = extract(Mat::size, GpuMat::size);
+   *
+ *

+ * + * @param ifCpu the function to call if this is backed by a mat in host memory + * @param ifGpu the function to call if this is backed by a mat in GPU memory + * @param the type of the property to extract + */ + private T extract(Function ifCpu, Function ifGpu) { + if (isCpu) { + return ifCpu.apply(cpuMat); + } else { + return ifGpu.apply(gpuMat); + } + } + + /** + * Gets the number of columns in this image. + */ + public int cols() { + return extract(Mat::cols, GpuMat::cols); + } + + /** + * Gets the number of rows in this image. + */ + public int rows() { + return extract(Mat::rows, GpuMat::rows); + } + + /** + * Gets the type of the data format of this image. + */ + public int type() { + return extract(Mat::type, GpuMat::type); + } + + /** + * Gets the number of color channels in this image. + */ + public int channels() { + return extract(Mat::channels, GpuMat::channels); + } + + /** + * Gets the channel depth of this image. + */ + public int depth() { + return extract(Mat::depth, GpuMat::depth); + } + + /** + * Checks if this image is empty. + */ + public boolean empty() { + return extract(Mat::empty, GpuMat::empty); + } + + /** + * Gets the size (width by height) of this image. + */ + public Size size() { + return extract(Mat::size, GpuMat::size); + } + + /** + * Gets the maximum possible value able to be held as a single element in this image. + */ + @SuppressWarnings("PMD") + public double highValue() { + return extract(Mat::highValue, g -> { + double highValue = 0.0; + switch (arrayDepth(g)) { + case IPL_DEPTH_8U: + highValue = 0xFF; + break; + case IPL_DEPTH_16U: + highValue = 0xFFFF; + break; + case IPL_DEPTH_8S: + highValue = Byte.MAX_VALUE; + break; + case IPL_DEPTH_16S: + highValue = Short.MAX_VALUE; + break; + case IPL_DEPTH_32S: + highValue = Integer.MAX_VALUE; + break; + case IPL_DEPTH_1U: + case IPL_DEPTH_32F: + case IPL_DEPTH_64F: + highValue = 1.0; + break; + default: + assert false; + } + return highValue; + }); + } + + private static int arrayDepth(GpuMat m) { + switch (m.depth()) { + case CV_8U: + return IPL_DEPTH_8U; + case CV_8S: + return IPL_DEPTH_8S; + case CV_16U: + return IPL_DEPTH_16U; + case CV_16S: + return IPL_DEPTH_16S; + case CV_32S: + return IPL_DEPTH_32S; + case CV_32F: + return IPL_DEPTH_32F; + case CV_64F: + return IPL_DEPTH_64F; + default: + throw new UnsupportedOperationException("Unsupported depth " + m.depth()); + } + } + + /** + * Allocates new array data if needed. + * + * @param rows New number of rows. + * @param cols New number of columns. + * @param type New matrix type. + */ + public void create(int rows, int cols, int type) { + if (isCpu) { + cpuMat.create(rows, cols, type); + } else { + gpuMat.create(rows, cols, type); + } + changed = true; + } + + /** + * Sets all or some of the array elements to the specified value. + * + * @param value Assigned scalar converted to the actual array type. + */ + public MatWrapper put(opencv_core.Scalar value) { + if (isCpu()) { + cpuMat.put(value); + } else { + gpuMat.setTo(value); + } + changed = true; + return this; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/AccelerationMode.java b/core/src/main/java/edu/wpi/grip/core/cuda/AccelerationMode.java new file mode 100644 index 0000000000..06a1b29fb1 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/AccelerationMode.java @@ -0,0 +1,13 @@ +package edu.wpi.grip.core.cuda; + +/** + * App-wide hardware acceleration mode. + */ +public interface AccelerationMode { + + /** + * Flag marking that GRIP is using CUDA-accelerated OpenCV. + */ + boolean isUsingCuda(); + +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/CudaAccelerationMode.java b/core/src/main/java/edu/wpi/grip/core/cuda/CudaAccelerationMode.java new file mode 100644 index 0000000000..215bf59d50 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/CudaAccelerationMode.java @@ -0,0 +1,8 @@ +package edu.wpi.grip.core.cuda; + +public class CudaAccelerationMode implements AccelerationMode { + @Override + public boolean isUsingCuda() { + return true; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/CudaDetector.java b/core/src/main/java/edu/wpi/grip/core/cuda/CudaDetector.java new file mode 100644 index 0000000000..57127d3ba5 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/CudaDetector.java @@ -0,0 +1,13 @@ +package edu.wpi.grip.core.cuda; + +/** + * Detects CUDA installs. + */ +public interface CudaDetector { + + /** + * Checks if a CUDA runtime is installed that is compatible with what we need for OpenCV. + */ + boolean isCompatibleCudaInstalled(); + +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/CudaVerifier.java b/core/src/main/java/edu/wpi/grip/core/cuda/CudaVerifier.java new file mode 100644 index 0000000000..ed61b13915 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/CudaVerifier.java @@ -0,0 +1,71 @@ +package edu.wpi.grip.core.cuda; + +import edu.wpi.grip.core.util.SafeShutdown; + +import com.google.common.annotations.VisibleForTesting; + +import java.util.Properties; +import java.util.logging.Logger; + +import javax.inject.Inject; +import javax.inject.Named; + +public class CudaVerifier { + + private static final Logger logger = Logger.getLogger(CudaVerifier.class.getName()); + + private static final String CUDA_VERSION_KEY = "edu.wpi.grip.cuda.version"; + + private final AccelerationMode accelerationMode; + private final CudaDetector cudaDetector; + private final String cudaVersion; + + @Inject + public CudaVerifier(AccelerationMode accelerationMode, + CudaDetector cudaDetector, + @Named("cudaProperties") Properties cudaProperties) { + this.accelerationMode = accelerationMode; + this.cudaDetector = cudaDetector; + this.cudaVersion = cudaProperties.getProperty(CUDA_VERSION_KEY, "Unknown"); + } + + /** + * Verifies the presence of a CUDA runtime, if required by the GRIP runtime, and exits the + * app if no compatible CUDA runtime is available. + */ + public void verifyCuda() { + if (!verify()) { + String message = "This version of GRIP requires CUDA version " + + cudaVersion + + " to be installed and an NVIDIA graphics card in your computer. " + + "If your computer does not have an NVIDIA graphics card, use a version of GRIP " + + "without CUDA acceleration. Otherwise, you need to install the appropriate CUDA " + + "runtime for your computer."; + logger.severe(message); + exit(); + } + } + + /** + * Verifies that, if GRIP is using CUDA acceleration, a compatible CUDA runtime is available. If + * GRIP is not using CUDA acceleration, this will always return {@code true}. + * + * @return false if GRIP is using CUDA acceleration but no compatible CUDA runtime is available, + * true otherwise + */ + public boolean verify() { + if (accelerationMode.isUsingCuda()) { + return cudaDetector.isCompatibleCudaInstalled(); + } else { + return true; + } + } + + /** + * Exits the application. + */ + @VisibleForTesting + void exit() { + SafeShutdown.exit(SafeShutdown.ExitCode.CUDA_UNAVAILABLE); + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/LoadingCudaDetector.java b/core/src/main/java/edu/wpi/grip/core/cuda/LoadingCudaDetector.java new file mode 100644 index 0000000000..f19ffbb84b --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/LoadingCudaDetector.java @@ -0,0 +1,36 @@ +package edu.wpi.grip.core.cuda; + +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.opencv_cudaarithm; + +/** + * Checks if CUDA is available by attempting to load one of the OpenCV CUDA class' JNI. If the JNI + * cannot be loaded, then no compatible CUDA runtime is available. This approach is probably the + * most flexible; it's OS-agnostic, since it lets the JVM handle loading the JNI libraries and + * linking, and doesn't require knowledge of CUDA installation locations - it just needs to be on + * the PATH. + * + *

Only one attempt is made to load the JNI, and the result is cached. Any later calls to + * {@link #isCompatibleCudaInstalled()} will simply return the cached value. + */ +public class LoadingCudaDetector implements CudaDetector { + + private volatile boolean hasCuda = false; + private volatile boolean checkedForCuda = false; + + @Override + public boolean isCompatibleCudaInstalled() { + if (!checkedForCuda) { + try { + Loader.load(opencv_cudaarithm.class); + hasCuda = true; + } catch (UnsatisfiedLinkError | NoClassDefFoundError e) { + // Couldn't load the JNI, no compatible CUDA runtime is available + hasCuda = false; + } + checkedForCuda = true; + } + + return hasCuda; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/NullAccelerationMode.java b/core/src/main/java/edu/wpi/grip/core/cuda/NullAccelerationMode.java new file mode 100644 index 0000000000..9272f7a4a2 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/NullAccelerationMode.java @@ -0,0 +1,8 @@ +package edu.wpi.grip.core.cuda; + +public class NullAccelerationMode implements AccelerationMode { + @Override + public boolean isUsingCuda() { + return false; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/cuda/NullCudaDetector.java b/core/src/main/java/edu/wpi/grip/core/cuda/NullCudaDetector.java new file mode 100644 index 0000000000..bb4ed1449d --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/cuda/NullCudaDetector.java @@ -0,0 +1,8 @@ +package edu.wpi.grip.core.cuda; + +public class NullCudaDetector implements CudaDetector { + @Override + public boolean isCompatibleCudaInstalled() { + return false; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java b/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java index b54ff29151..8897e9b1f4 100644 --- a/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java +++ b/core/src/main/java/edu/wpi/grip/core/events/UnexpectedThrowableEvent.java @@ -55,7 +55,7 @@ public void handleSafely(UnexpectedThrowableEventHandler handler) { try { logger.log(Level.SEVERE, "Failed to handle safely", throwable); } finally { - SafeShutdown.exit(1); + SafeShutdown.exit(SafeShutdown.ExitCode.MISC_ERROR); } } finally { shutdownIfFatal(); @@ -72,7 +72,7 @@ public void shutdownIfFatal() { logger.log(Level.SEVERE, "Shutting down from error", throwable); } finally { // If all else fails then shutdown - SafeShutdown.exit(1); + SafeShutdown.exit(SafeShutdown.ExitCode.MISC_ERROR); } } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java b/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java index af12ad0da7..7df0d45a75 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/CVOperations.java @@ -1,6 +1,5 @@ package edu.wpi.grip.core.operations; - import edu.wpi.grip.core.OperationMetaData; import edu.wpi.grip.core.events.OperationAddedEvent; import edu.wpi.grip.core.operations.opencv.CVOperation; @@ -24,16 +23,48 @@ import com.google.common.eventbus.EventBus; import com.google.inject.Inject; -import org.bytedeco.javacpp.opencv_core; import org.bytedeco.javacpp.opencv_core.Point; import org.bytedeco.javacpp.opencv_core.Scalar; import org.bytedeco.javacpp.opencv_core.Size; +import org.bytedeco.javacpp.opencv_cudaarithm; +import org.bytedeco.javacpp.opencv_cudafilters.Filter; +import org.bytedeco.javacpp.opencv_cudaimgproc; import org.bytedeco.javacpp.opencv_imgproc; +import static org.bytedeco.javacpp.opencv_core.absdiff; +import static org.bytedeco.javacpp.opencv_core.add; +import static org.bytedeco.javacpp.opencv_core.addWeighted; +import static org.bytedeco.javacpp.opencv_core.bitwise_and; +import static org.bytedeco.javacpp.opencv_core.bitwise_not; +import static org.bytedeco.javacpp.opencv_core.bitwise_or; +import static org.bytedeco.javacpp.opencv_core.bitwise_xor; +import static org.bytedeco.javacpp.opencv_core.compare; +import static org.bytedeco.javacpp.opencv_core.divide; +import static org.bytedeco.javacpp.opencv_core.extractChannel; +import static org.bytedeco.javacpp.opencv_core.flip; +import static org.bytedeco.javacpp.opencv_core.max; +import static org.bytedeco.javacpp.opencv_core.min; +import static org.bytedeco.javacpp.opencv_core.multiply; +import static org.bytedeco.javacpp.opencv_core.scaleAdd; +import static org.bytedeco.javacpp.opencv_core.subtract; +import static org.bytedeco.javacpp.opencv_core.transpose; +import static org.bytedeco.javacpp.opencv_cudafilters.createSobelFilter; +import static org.bytedeco.javacpp.opencv_imgproc.GaussianBlur; +import static org.bytedeco.javacpp.opencv_imgproc.Laplacian; +import static org.bytedeco.javacpp.opencv_imgproc.Sobel; +import static org.bytedeco.javacpp.opencv_imgproc.adaptiveThreshold; +import static org.bytedeco.javacpp.opencv_imgproc.applyColorMap; +import static org.bytedeco.javacpp.opencv_imgproc.cvtColor; +import static org.bytedeco.javacpp.opencv_imgproc.dilate; +import static org.bytedeco.javacpp.opencv_imgproc.medianBlur; +import static org.bytedeco.javacpp.opencv_imgproc.rectangle; +import static org.bytedeco.javacpp.opencv_imgproc.resize; +import static org.bytedeco.javacpp.opencv_imgproc.threshold; + /** * A list of all of the raw opencv operations. */ -@SuppressWarnings("PMD.AvoidDuplicateLiterals") +@SuppressWarnings({"PMD.AvoidDuplicateLiterals", "CodeBlock2Expr"}) public class CVOperations { private final EventBus eventBus; @@ -47,138 +78,196 @@ public class CVOperations { this.coreOperations = ImmutableList.of( new OperationMetaData(CVOperation.defaults("CV absdiff", "Calculate the per-element absolute difference of two images."), - templateFactory.createAllMatTwoSource(opencv_core::absdiff)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.absdiff(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + absdiff(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV add", "Calculate the per-pixel sum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::add)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.add(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + add(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV addWeighted", "Calculate the weighted sum of two images."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src1"), SocketHints.Inputs.createNumberSpinnerSocketHint("alpha", 0), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("beta", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("gamma", 0), - SocketHints.Outputs.createMatSocketHint("dst"), - (src1, alpha, src2, beta, gamma, dst) -> { - opencv_core.addWeighted(src1, alpha.doubleValue(), src2, beta.doubleValue(), - gamma.doubleValue(), dst); + SocketHints.createImageSocketHint("dst"), + (src1, alpha, src2, beta, gamma, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.addWeighted(src1.getGpu(), alpha.doubleValue(), src2.getGpu(), + beta.doubleValue(), gamma.doubleValue(), dst.rawGpu()); + } else { + addWeighted(src1.getCpu(), alpha.doubleValue(), src2.getCpu(), + beta.doubleValue(), gamma.doubleValue(), dst.rawCpu()); + } } )), new OperationMetaData(CVOperation.defaults("CV bitwise_and", "Calculate the per-element bitwise conjunction of two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_and)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_and(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + bitwise_and(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV bitwise_not", "Calculate per-element bit-wise inversion of an image."), - templateFactory.createAllMatOneSource(opencv_core::bitwise_not)), + templateFactory.createAllMatOneSourceCuda((src, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_not(src.getGpu(), dst.rawGpu()); + } else { + bitwise_not(src.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV bitwise_or", "Calculate the per-element bit-wise disjunction of two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_or)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_or(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + bitwise_or(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV bitwise_xor", "Calculate the per-element bit-wise \"exclusive or\" on two images."), - templateFactory.createAllMatTwoSource(opencv_core::bitwise_xor)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.bitwise_xor(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + bitwise_xor(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV compare", "Compare each pixel in two images using a given rule."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.createEnumSocketHint("cmpop", CmpTypesEnum.CMP_EQ), - SocketHints.Outputs.createMatSocketHint("dst"), - (src1, src2, cmp, dst) -> { - opencv_core.compare(src1, src2, dst, cmp.value); + SocketHints.createImageSocketHint("dst"), + (src1, src2, cmp, useCuda, dst) -> { + int cmpop = cmp.value; + if (useCuda) { + opencv_cudaarithm.compare(src1.getGpu(), src2.getGpu(), dst.rawGpu(), cmpop); + } else { + compare(src1.getCpu(), src2.getCpu(), dst.rawCpu(), cmpop); + } } )), new OperationMetaData(CVOperation.defaults("CV divide", "Perform per-pixel division of two images."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0, -Double.MAX_VALUE, Double.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, src2, scale, dst) -> { - opencv_core.divide(src1, src2, dst, scale.doubleValue(), -1); + divide(src1.getCpu(), src2.getCpu(), dst.rawCpu(), scale.doubleValue(), -1); } )), new OperationMetaData(CVOperation.defaults("CV extractChannel", "Extract a single channel from a image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("channel", 0, 0, Integer .MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, coi, dst) -> { - opencv_core.extractChannel(src1, dst, coi.intValue()); + extractChannel(src1.getCpu(), dst.rawCpu(), coi.intValue()); } )), new OperationMetaData(CVOperation.defaults("CV flip", "Flip image around vertical, horizontal, or both axes."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("flipCode", FlipCode.Y_AXIS), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, flipCode, dst) -> { - opencv_core.flip(src, dst, flipCode.value); + flip(src.getCpu(), dst.rawCpu(), flipCode.value); } )), new OperationMetaData(CVOperation.defaults("CV max", "Calculate per-element maximum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::max)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + max(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV min", "Calculate the per-element minimum of two images."), - templateFactory.createAllMatTwoSource(opencv_core::min)), + templateFactory.createAllMatTwoSource((src1, src2, dst) -> { + min(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + })), new OperationMetaData(CVOperation.defaults("CV multiply", "Calculate the per-pixel scaled product of two images."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), - SocketHints.Inputs.createMatSocketHint("src2", false), + SocketHints.createImageSocketHint("src1"), + SocketHints.createImageSocketHint("src2"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0, Integer.MIN_VALUE, Integer.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src1, src2, scale, dst) -> { - opencv_core.multiply(src1, src2, dst, scale.doubleValue(), -1); + multiply(src1.getCpu(), src2.getCpu(), dst.getCpu(), scale.doubleValue(), -1); } )), new OperationMetaData(CVOperation.defaults("CV scaleAdd", "Calculate the sum of two images where one image is multiplied by a scalar."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src1", false), + SocketHints.createImageSocketHint("src1"), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0), - SocketHints.Inputs.createMatSocketHint("src2", false), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("src2"), + SocketHints.createImageSocketHint("dst"), (src1, alpha, src2, dst) -> { - opencv_core.scaleAdd(src1, alpha.doubleValue(), src2, dst); + scaleAdd(src1.getCpu(), alpha.doubleValue(), src2.getCpu(), dst.rawCpu()); } )), new OperationMetaData(CVOperation.defaults("CV subtract", "Calculate the per-pixel difference between two images."), - templateFactory.createAllMatTwoSource(opencv_core::subtract)), + templateFactory.createAllMatTwoSourceCuda((src1, src2, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.subtract(src1.getGpu(), src2.getGpu(), dst.rawGpu()); + } else { + subtract(src1.getCpu(), src2.getCpu(), dst.rawCpu()); + } + })), new OperationMetaData(CVOperation.defaults("CV transpose", "Calculate the transpose of an image."), - templateFactory.createAllMatOneSource(opencv_core::transpose)) + templateFactory.createAllMatOneSource((src, dst) -> { + transpose(src.getCpu(), dst.rawCpu()); + })) ); this.imgprocOperation = ImmutableList.of( new OperationMetaData(CVOperation.defaults("CV adaptiveThreshold", "Transforms a grayscale image to a binary image)."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("maxValue", 0.0), SocketHints.createEnumSocketHint("adaptiveMethod", AdaptiveThresholdTypesEnum.ADAPTIVE_THRESH_MEAN_C), @@ -186,9 +275,9 @@ public class CVOperations { CVAdaptThresholdTypesEnum.THRESH_BINARY), SocketHints.Inputs.createNumberSpinnerSocketHint("blockSize", 0.0), SocketHints.Inputs.createNumberSpinnerSocketHint("C", 0.0), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, maxValue, adaptiveMethod, thresholdType, blockSize, c, dst) -> { - opencv_imgproc.adaptiveThreshold(src, dst, maxValue.doubleValue(), + adaptiveThreshold(src.getCpu(), dst.rawCpu(), maxValue.doubleValue(), adaptiveMethod.value, thresholdType.value, blockSize.intValue(), c .doubleValue()); } @@ -197,54 +286,43 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV applyColorMap", "Apply a MATLAB equivalent colormap to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("colormap", ColormapTypesEnum.COLORMAP_AUTUMN), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, colormap, dst) -> { - opencv_imgproc.applyColorMap(src, dst, colormap.value); - } - )), - - new OperationMetaData(CVOperation.defaults("CV Canny", - "Apply a \"canny edge detection\" algorithm to an image."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("image", false), - SocketHints.Inputs.createNumberSpinnerSocketHint("threshold1", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("threshold2", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("apertureSize", 3), - SocketHints.Inputs.createCheckboxSocketHint("L2gradient", false), - SocketHints.Outputs.createMatSocketHint("edges"), - (image, threshold1, threshold2, apertureSize, l2gradient, edges) -> { - opencv_imgproc.Canny(image, edges, threshold1.doubleValue(), threshold2 - .doubleValue(), apertureSize.intValue(), l2gradient); + applyColorMap(src.getCpu(), dst.rawCpu(), colormap.value); } )), new OperationMetaData(CVOperation.defaults("CV cvtColor", "Convert an image from one color space to another."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src"), SocketHints.createEnumSocketHint("code", ColorConversionCodesEnum.COLOR_BGR2BGRA), - SocketHints.Outputs.createMatSocketHint("dst"), - (src, code, dst) -> { - opencv_imgproc.cvtColor(src, dst, code.value); + SocketHints.createImageSocketHint("dst"), + (src, code, useCuda, dst) -> { + if (useCuda) { + opencv_cudaimgproc.cvtColor(dst.getGpu(), dst.rawGpu(), code.value); + } else { + cvtColor(src.getCpu(), dst.rawCpu(), code.value); + } } )), new OperationMetaData(CVOperation.defaults("CV dilate", "Expands areas of higher values in an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), - SocketHints.Inputs.createMatSocketHint("kernel", true), + SocketHints.createImageSocketHint("src"), + SocketHints.createImageSocketHint("kernel"), new SocketHint.Builder<>(Point.class).identifier("anchor").initialValueSupplier( () -> new Point(-1, -1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("iterations", 1), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_CONSTANT), new SocketHint.Builder<>(Scalar.class).identifier("borderValue") .initialValueSupplier(opencv_imgproc::morphologyDefaultBorderValue).build(), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, kernel, anchor, iterations, borderType, borderValue, dst) -> { - opencv_imgproc.dilate(src, dst, kernel, anchor, iterations.intValue(), + dilate(src.getCpu(), dst.rawCpu(), kernel.getCpu(), anchor, iterations.intValue(), borderType.value, borderValue); } )), @@ -252,33 +330,33 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV erode", "Expands areas of lower values in an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), - SocketHints.Inputs.createMatSocketHint("kernel", true), + SocketHints.createImageSocketHint("src"), + SocketHints.createImageSocketHint("kernel"), new SocketHint.Builder<>(Point.class).identifier("anchor").initialValueSupplier( () -> new Point(-1, -1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("iterations", 1), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_CONSTANT), new SocketHint.Builder<>(Scalar.class).identifier("borderValue") .initialValueSupplier(opencv_imgproc::morphologyDefaultBorderValue).build(), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, kernel, anchor, iterations, borderType, borderValue, dst) -> { - opencv_imgproc.erode(src, dst, kernel, anchor, iterations.intValue(), - borderType.value, borderValue); + opencv_imgproc.erode(src.getCpu(), dst.rawCpu(), kernel.getCpu(), anchor, + iterations.intValue(), borderType.value, borderValue); } )), new OperationMetaData(CVOperation.defaults("CV GaussianBlur", "Apply a Gaussian blur to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", true), + SocketHints.createImageSocketHint("src"), new SocketHint.Builder<>(Size.class).identifier("ksize").initialValueSupplier(() -> new Size(1, 1)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaX", 0.0), - SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaY", 0.0), SocketHints - .createEnumSocketHint("borderType", CVBorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.Inputs.createNumberSpinnerSocketHint("sigmaY", 0.0), + SocketHints.createEnumSocketHint("borderType", CVBorderTypesEnum.BORDER_DEFAULT), + SocketHints.createImageSocketHint("dst"), (src, ksize, sigmaX, sigmaY, borderType, dst) -> { - opencv_imgproc.GaussianBlur(src, dst, ksize, sigmaX.doubleValue(), sigmaY + GaussianBlur(src.getCpu(), dst.rawCpu(), ksize, sigmaX.doubleValue(), sigmaY .doubleValue(), borderType.value); } )), @@ -286,14 +364,14 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV Laplacian", "Find edges by calculating the Laplacian for the given image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 1), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1.0), SocketHints.Inputs.createNumberSpinnerSocketHint("delta", 0.0), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, ksize, scale, delta, borderType, dst) -> { - opencv_imgproc.Laplacian(src, dst, 0, ksize.intValue(), scale.doubleValue(), + Laplacian(src.getCpu(), dst.rawCpu(), 0, ksize.intValue(), scale.doubleValue(), delta.doubleValue(), borderType.value); } )), @@ -301,18 +379,18 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV medianBlur", "Apply a Median blur to an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 1, 1, Integer.MAX_VALUE), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, ksize, dst) -> { - opencv_imgproc.medianBlur(src, dst, ksize.intValue()); + medianBlur(src.getCpu(), dst.rawCpu(), ksize.intValue()); } )), new OperationMetaData(CVOperation.defaults("CV rectangle", "Draw a rectangle (outline or filled) on an image."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createPointSocketHint("pt1", 0, 0), SocketHints.Inputs.createPointSocketHint("pt2", 0, 0), new SocketHint.Builder<>(Scalar.class).identifier("color").initialValueSupplier( @@ -321,12 +399,12 @@ public class CVOperations { .MIN_VALUE, Integer.MAX_VALUE), SocketHints.createEnumSocketHint("lineType", LineTypesEnum.LINE_8), SocketHints.Inputs.createNumberSpinnerSocketHint("shift", 0), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, pt1, pt2, color, thickness, lineType, shift, dst) -> { // Rectangle only has one input and it modifies it so we have to copy the input // image to the dst src.copyTo(dst); - opencv_imgproc.rectangle(dst, pt1, pt2, color, thickness.intValue(), lineType + rectangle(dst.rawCpu(), pt1, pt2, color, thickness.intValue(), lineType .value, shift.intValue()); } )), @@ -334,49 +412,74 @@ public class CVOperations { new OperationMetaData(CVOperation.defaults("CV resize", "Resizes the image to the specified size."), templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + SocketHints.createImageSocketHint("src"), new SocketHint.Builder<>(Size.class).identifier("dsize").initialValueSupplier(() -> new Size(0, 0)).build(), SocketHints.Inputs.createNumberSpinnerSocketHint("fx", .25), SocketHints.Inputs .createNumberSpinnerSocketHint("fy", .25), SocketHints.createEnumSocketHint("interpolation", InterpolationFlagsEnum .INTER_LINEAR), - SocketHints.Outputs.createMatSocketHint("dst"), + SocketHints.createImageSocketHint("dst"), (src, dsize, fx, fy, interpolation, dst) -> { - opencv_imgproc.resize(src, dst, dsize, fx.doubleValue(), fy.doubleValue(), + resize(src.getCpu(), dst.rawCpu(), dsize, fx.doubleValue(), fy.doubleValue(), interpolation.value); } )), new OperationMetaData(CVOperation.defaults("CV Sobel", "Find edges by calculating the requested derivative order for the given image."), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("dx", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("dy", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("ksize", 3), SocketHints.Inputs.createNumberSpinnerSocketHint("scale", 1), SocketHints.Inputs.createNumberSpinnerSocketHint("delta", 0), SocketHints.createEnumSocketHint("borderType", BorderTypesEnum.BORDER_DEFAULT), - SocketHints.Outputs.createMatSocketHint("dst"), - (src, dx, dy, ksize, scale, delta, borderType, dst) -> { - opencv_imgproc.Sobel(src, dst, 0, dx.intValue(), dy.intValue(), - ksize.intValue(), scale.doubleValue(), delta.doubleValue(), borderType.value); + SocketHints.createImageSocketHint("dst"), + (src, dx, dy, ksize, scale, delta, borderType, useCuda, dst) -> { + if (useCuda) { + try (Filter sobelFilter = createSobelFilter( + src.type(), + src.type(), + dx.intValue(), + dy.intValue(), + ksize.intValue(), + scale.doubleValue(), + borderType.value, + borderType.value)) { + sobelFilter.apply(src.getGpu(), dst.rawGpu()); + } + } else { + Sobel(src.getCpu(), dst.rawCpu(), 0, dx.intValue(), dy.intValue(), + ksize.intValue(), scale.doubleValue(), delta.doubleValue(), + borderType.value); + } } )), new OperationMetaData(CVOperation.defaults("CV Threshold", "Apply a fixed-level threshold to each array element in an image.", "CV threshold"), - templateFactory.create( - SocketHints.Inputs.createMatSocketHint("src", false), + templateFactory.createCuda( + SocketHints.createImageSocketHint("src"), SocketHints.Inputs.createNumberSpinnerSocketHint("thresh", 0), SocketHints.Inputs.createNumberSpinnerSocketHint("maxval", 0), SocketHints.createEnumSocketHint("type", CVThresholdTypesEnum.THRESH_BINARY), - SocketHints.Outputs.createMatSocketHint("dst"), - (src, thresh, maxval, type, dst) -> { - opencv_imgproc.threshold(src, dst, thresh.doubleValue(), maxval.doubleValue(), - type.value); + SocketHints.createImageSocketHint("dst"), + (src, thresh, maxval, type, useCuda, dst) -> { + if (useCuda) { + opencv_cudaarithm.threshold( + src.getGpu(), + dst.rawGpu(), + thresh.doubleValue(), + maxval.doubleValue(), + type.value + ); + } else { + threshold(src.getCpu(), dst.rawCpu(), thresh.doubleValue(), + maxval.doubleValue(), type.value); + } } )) ); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java new file mode 100644 index 0000000000..a6e572a3d6 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/CudaOperation.java @@ -0,0 +1,85 @@ +package edu.wpi.grip.core.operations; + +import edu.wpi.grip.core.MatWrapper; +import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.sockets.CudaSocket; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; +import edu.wpi.grip.core.sockets.SocketHints; + +import org.bytedeco.javacpp.opencv_core.GpuMat; +import org.bytedeco.javacpp.opencv_core.Mat; + +/** + * A partial implementation of Operation that has the option to use CUDA acceleration. + */ +public abstract class CudaOperation implements Operation { + + protected final SocketHint inputHint = + SocketHints.createImageSocketHint("Input"); + protected final SocketHint gpuHint = + SocketHints.createBooleanSocketHint("Prefer GPU", false); + protected final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); + + /** + * Default image input socket. + */ + protected final InputSocket inputSocket; + /** + * Input socket telling the operation to prefer to use CUDA acceleration when possible. + */ + protected final CudaSocket gpuSocket; + /** + * Default image output socket. + */ + protected final OutputSocket outputSocket; + + /** + * The mat used for an input to the CPU operation. + */ + protected final Mat cpuIn = new Mat(); + + /** + * The mat used for an input to the CUDA operation. + */ + protected final GpuMat gpuIn = new GpuMat(); + + /** + * The output mat of a CPU operation. + */ + protected final Mat cpuOut = new Mat(); + + /** + * The output mat of a CUDA operation. + */ + protected final GpuMat gpuOut = new GpuMat(); + + protected CudaOperation(InputSocket.Factory isf, OutputSocket.Factory osf) { + inputSocket = isf.create(inputHint); + gpuSocket = isf.createCuda(gpuHint); + outputSocket = osf.create(outputHint); + + inputSocket.setValue(MatWrapper.using(cpuIn, gpuIn)); + outputSocket.setValue(MatWrapper.using(cpuOut, gpuOut)); + } + + @Override + public void cleanUp() { + cpuIn.deallocate(); + gpuIn.deallocate(); + cpuOut.deallocate(); + gpuOut.deallocate(); + } + + /** + * Checks the {@link #gpuSocket} to see if this operation should prefer to use the CUDA codepath. + * + * @return true if this operation should prefer to use CUDA, false if it should only use the CPU + */ + protected boolean preferCuda() { + return gpuSocket.isCudaAvailable() + && gpuSocket.getValue().orElse(false); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java index 4671533e8c..881c3b16d9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlobsReport.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; @@ -10,25 +11,23 @@ import java.util.Collections; import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; - /** * This class is used as the output of operations that detect blobs in an image. */ @PublishableObject @NoSocketTypeLabel public class BlobsReport implements Publishable { - private final Mat input; + private final MatWrapper input; private final List blobs; /** * Create an empty blob report. This is used as the default value for sockets */ public BlobsReport() { - this(new Mat(), Collections.emptyList()); + this(MatWrapper.emptyWrapper(), Collections.emptyList()); } - public BlobsReport(Mat input, List blobs) { + public BlobsReport(MatWrapper input, List blobs) { this.input = input; this.blobs = blobs; } @@ -40,7 +39,7 @@ public List getBlobs() { /** * @return The original image that the blob detection was performed on. */ - public Mat getInput() { + public MatWrapper getInput() { return this.input; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java index 15bc184d7a..326bf7b9c6 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/BlurOperation.java @@ -2,7 +2,9 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -11,10 +13,18 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_core.GpuMat; +import org.bytedeco.javacpp.opencv_cudaimgproc; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; +import static org.bytedeco.javacpp.opencv_core.CV_8UC3; +import static org.bytedeco.javacpp.opencv_core.CV_8UC4; import static org.bytedeco.javacpp.opencv_core.Size; +import static org.bytedeco.javacpp.opencv_cudafilters.Filter; +import static org.bytedeco.javacpp.opencv_cudafilters.createGaussianFilter; +import static org.bytedeco.javacpp.opencv_imgproc.CV_BGR2BGRA; +import static org.bytedeco.javacpp.opencv_imgproc.CV_BGRA2BGR; import static org.bytedeco.javacpp.opencv_imgproc.GaussianBlur; import static org.bytedeco.javacpp.opencv_imgproc.bilateralFilter; import static org.bytedeco.javacpp.opencv_imgproc.blur; @@ -27,27 +37,27 @@ summary = "Blurs an image to remove noise", category = OperationCategory.IMAGE_PROCESSING, iconName = "blur") -public class BlurOperation implements Operation { +public class BlurOperation extends CudaOperation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.BOX); private final SocketHint radiusHint = SocketHints.Inputs .createNumberSliderSocketHint("Radius", 0.0, 0.0, 100.0); - private final SocketHint outputHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket inputSocket; private final InputSocket typeSocket; private final InputSocket radiusSocket; - private final OutputSocket outputSocket; + + private int lastKernelSize = 0; + // used to covert 3-channel images to 4-channel for CUDA + private final GpuMat upcast = new GpuMat(); + private Filter gpuGaussianFilter; + //private Filter gpuMedianFilter; @Inject @SuppressWarnings("JavadocMethod") - public BlurOperation(InputSocket.Factory inputSocketFactory, - OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(inputHint); + public BlurOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory + outputSocketFactory) { + super(inputSocketFactory, outputSocketFactory); this.typeSocket = inputSocketFactory.create(typeHint); this.radiusSocket = inputSocketFactory.create(radiusHint); - - this.outputSocket = outputSocketFactory.create(outputHint); } @Override @@ -55,7 +65,8 @@ public List getInputSockets() { return ImmutableList.of( inputSocket, typeSocket, - radiusSocket + radiusSocket, + gpuSocket ); } @@ -68,47 +79,115 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); + if (input.empty()) { + return; + } final Type type = typeSocket.getValue().get(); final Number radius = radiusSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final MatWrapper output = outputSocket.getValue().get(); + int imageType; int kernelSize; + boolean kernelChange; + + if (preferCuda()) { + if (input.type() == CV_8UC3) { + // GPU filters generally don't take BGR images, but will take BGRA + // So we convert the BGR image to BGRA here and convert it back at the end + // Note that this doesn't care about the actual pixel format because we're + // converting to the same format, just with an extra channel + imageType = CV_8UC4; + opencv_cudaimgproc.cvtColor(input.getGpu(), upcast, CV_BGR2BGRA); + gpuIn.put(upcast); + } else { + input.copyTo(gpuIn); + imageType = input.type(); + } + } else { + imageType = input.type(); + } switch (type) { case BOX: // Box filter kernels must have an odd size kernelSize = 2 * radius.intValue() + 1; - blur(input, output, new Size(kernelSize, kernelSize)); + + // Don't bother with CUDA acceleration here; CPU is fast enough that memory copies + // will remove the speedups from CUDA + blur(input.getCpu(), output.getCpu(), new Size(kernelSize, kernelSize)); break; case GAUSSIAN: // A Gaussian blur radius is a standard deviation, so a kernel that extends three radii - // in either direction - // from the center should account for 99.7% of the theoretical influence on each pixel. + // in either direction from the center should account for 99.7% of the theoretical + // influence on each pixel. kernelSize = 6 * radius.intValue() + 1; - GaussianBlur(input, output, new Size(kernelSize, kernelSize), radius.doubleValue()); + kernelChange = kernelSize != lastKernelSize; + lastKernelSize = kernelSize; + if (preferCuda()/* && kernelSize < 32*/) { + // GPU gaussian blurs require kernel size in 0..31 + if (kernelChange || gpuGaussianFilter == null) { + gpuGaussianFilter = createGaussianFilter(imageType, imageType, + new Size(kernelSize, kernelSize), radius.doubleValue()); + } + gpuGaussianFilter.apply(gpuIn, gpuOut); + output.set(gpuOut); + } else { + GaussianBlur(input.getCpu(), output.getCpu(), new Size(kernelSize, kernelSize), + radius.doubleValue()); + } break; case MEDIAN: kernelSize = 2 * radius.intValue() + 1; - medianBlur(input, output, kernelSize); + // FIXME: CUDA median filters is broken - run on CPU only for now + /* + kernelChange = kernelSize != lastKernelSize; + lastKernelSize = kernelSize; + if (preferCuda() && imageType == CV_8UC1) { + // GPU median filters only work on grayscale images + if (kernelChange || gpuMedianFilter == null) { + gpuMedianFilter = createMedianFilter(imageType, kernelSize); + } + gpuMedianFilter.apply(gpuIn, gpuOut); + output.set(gpuOut); + } else { + medianBlur(input.getCpu(), output.rawCpu(), kernelSize); + } + */ + medianBlur(input.getCpu(), output.rawCpu(), kernelSize); break; case BILATERAL_FILTER: - bilateralFilter(input, output, -1, radius.doubleValue(), radius.doubleValue()); + if (preferCuda()) { + opencv_cudaimgproc.bilateralFilter(gpuIn, gpuOut, + -1, radius.floatValue(), radius.floatValue() / 6); + output.set(gpuOut); + } else { + bilateralFilter(input.getCpu(), output.rawCpu(), + -1, radius.doubleValue(), radius.doubleValue() / 6); + } break; default: throw new IllegalArgumentException("Illegal blur type: " + type); } + if (preferCuda() && output.type() == CV_8UC4 && input.type() == CV_8UC3) { + // Remove the alpha channel that was added for GPU filtering + opencv_cudaimgproc.cvtColor(output.getGpu(), output.rawGpu(), CV_BGRA2BGR); + } + + //output.set(output); outputSocket.setValue(output); } private enum Type { - BOX("Box Blur"), GAUSSIAN("Gaussian Blur"), MEDIAN("Median Filter"), + BOX("Box Blur"), + GAUSSIAN("Gaussian Blur"), + MEDIAN("Median Filter"), BILATERAL_FILTER("Bilateral Filter"); private final String label; diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java new file mode 100644 index 0000000000..52c52fe124 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/CannyEdgeOperation.java @@ -0,0 +1,93 @@ +package edu.wpi.grip.core.operations.composite; + +import edu.wpi.grip.annotation.operation.Description; +import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; +import edu.wpi.grip.core.sockets.SocketHints; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; + +import org.bytedeco.javacpp.opencv_cudaimgproc.CannyEdgeDetector; + +import java.util.List; + +import static org.bytedeco.javacpp.opencv_cudaimgproc.createCannyEdgeDetector; +import static org.bytedeco.javacpp.opencv_imgproc.Canny; + +/** + * An operation that performs canny edge detection on an image. + */ +@Description(name = "CV Canny", + summary = "Performs canny edge detection on a grayscale image", + category = OperationCategory.OPENCV, + iconName = "opencv") +public class CannyEdgeOperation extends CudaOperation { + + private final SocketHint lowThreshHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("Low threshold", 0); + private final SocketHint highThreshHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("High threshold", 0); + private final SocketHint apertureSizeHint + = SocketHints.Inputs.createNumberSpinnerSocketHint("Aperture size", 0); + private final SocketHint l2gradientHint + = SocketHints.Inputs.createCheckboxSocketHint("L2gradient", false); + + private final InputSocket lowThreshSocket; + private final InputSocket highThreshSocket; + private final InputSocket apertureSizeSocket; + private final InputSocket l2gradientSocket; + + @Inject + protected CannyEdgeOperation(InputSocket.Factory isf, OutputSocket.Factory osf) { + super(isf, osf); + lowThreshSocket = isf.create(lowThreshHint); + highThreshSocket = isf.create(highThreshHint); + apertureSizeSocket = isf.create(apertureSizeHint); + l2gradientSocket = isf.create(l2gradientHint); + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + inputSocket, + lowThreshSocket, + highThreshSocket, + apertureSizeSocket, + l2gradientSocket, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + outputSocket + ); + } + + @Override + public void perform() { + double lowThresh = lowThreshSocket.getValue().get().doubleValue(); + double highThresh = highThreshSocket.getValue().get().doubleValue(); + int apertureSize = apertureSizeSocket.getValue().get().intValue(); + boolean l2gradient = l2gradientSocket.getValue().get(); + if (preferCuda()) { + try (CannyEdgeDetector cannyEdgeDetector = createCannyEdgeDetector( + lowThresh, + highThresh, + apertureSize, + l2gradient)) { + cannyEdgeDetector.detect(inputSocket.getValue().get().getGpu(), + outputSocket.getValue().get().rawGpu()); + } + } else { + Canny(inputSocket.getValue().get().getCpu(), outputSocket.getValue().get().getCpu(), + lowThresh, highThresh, apertureSize, l2gradient); + } + outputSocket.flagChanged(); + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java index e24bf84308..0bca69bd7d 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -29,8 +30,8 @@ iconName = "opencv") public class CascadeClassifierOperation implements Operation { - private final SocketHint imageHint = - SocketHints.Inputs.createMatSocketHint("Image", false); + private final SocketHint imageHint = + SocketHints.createImageSocketHint("Image"); private final SocketHint classifierHint = new SocketHint.Builder<>(CascadeClassifier.class) .identifier("Classifier") @@ -49,7 +50,7 @@ public class CascadeClassifierOperation implements Operation { .initialValue(RectsReport.NIL) .build(); - private final InputSocket imageSocket; + private final InputSocket imageSocket; private final InputSocket classifierSocket; private final InputSocket scaleSocket; private final InputSocket minNeighborsSocket; @@ -93,7 +94,8 @@ public void perform() { if (!imageSocket.getValue().isPresent() || !classifierSocket.getValue().isPresent()) { return; } - final Mat image = imageSocket.getValue().get(); + final MatWrapper input = imageSocket.getValue().get(); + final Mat image = input.getCpu(); if (image.empty() || image.channels() != 3) { throw new IllegalArgumentException("A cascade classifier needs a three-channel input"); } @@ -108,7 +110,7 @@ public void perform() { for (int i = 0; i < detections.size(); i++) { rects.add(detections.get(i)); } - output.setValue(new RectsReport(image, rects)); + output.setValue(new RectsReport(input, rects)); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java index 59ee2e7c17..00ea135388 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/DesaturateOperation.java @@ -2,18 +2,19 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; -import edu.wpi.grip.core.sockets.SocketHint; -import edu.wpi.grip.core.sockets.SocketHints; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_cudaimgproc; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgproc.COLOR_BGR2GRAY; import static org.bytedeco.javacpp.opencv_imgproc.COLOR_BGRA2GRAY; import static org.bytedeco.javacpp.opencv_imgproc.cvtColor; @@ -25,27 +26,21 @@ summary = "Convert a color image into shades of gray", category = OperationCategory.IMAGE_PROCESSING, iconName = "desaturate") -public class DesaturateOperation implements Operation { - - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); - - private final InputSocket inputSocket; - private final OutputSocket outputSocket; +public class DesaturateOperation extends CudaOperation { @Inject @SuppressWarnings("JavadocMethod") public DesaturateOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(inputHint); - this.outputSocket = outputSocketFactory.create(outputHint); + super(inputSocketFactory, outputSocketFactory); } @Override public List getInputSockets() { return ImmutableList.of( - inputSocket + inputSocket, + gpuSocket ); } @@ -58,23 +53,30 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); - - Mat output = outputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); + final MatWrapper output = outputSocket.getValue().get(); switch (input.channels()) { case 1: // If the input is already one channel, it's already desaturated - input.copyTo(output); + output.set(input); break; case 3: - cvtColor(input, output, COLOR_BGR2GRAY); + if (preferCuda()) { + opencv_cudaimgproc.cvtColor(input.getGpu(), output.rawGpu(), COLOR_BGR2GRAY); + } else { + cvtColor(input.getCpu(), output.rawCpu(), COLOR_BGR2GRAY); + } break; case 4: - cvtColor(input, output, COLOR_BGRA2GRAY); + if (preferCuda()) { + opencv_cudaimgproc.cvtColor(input.getGpu(), output.rawGpu(), COLOR_BGRA2GRAY); + } else { + cvtColor(input.getCpu(), output.rawCpu(), COLOR_BGRA2GRAY); + } break; default: diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java index 17c52ab339..0837d54680 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/DistanceTransformOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -31,15 +32,15 @@ iconName = "opencv") public class DistanceTransformOperation implements Operation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint srcHint = SocketHints.createImageSocketHint("Input"); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.DIST_L2); private final SocketHint maskSizeHint = SocketHints.createEnumSocketHint("Mask size", MaskSize.ZERO); - private final SocketHint outputHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket srcSocket; + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); + private final InputSocket srcSocket; private final InputSocket typeSocket; private final InputSocket maskSizeSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -70,7 +71,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); + final Mat input = srcSocket.getValue().get().getCpu(); if (input.type() != CV_8U) { throw new IllegalArgumentException("Distance transform only works on 8-bit binary images"); @@ -79,12 +80,12 @@ public void perform() { final Type type = typeSocket.getValue().get(); final MaskSize maskSize = maskSizeSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); distanceTransform(input, output, type.value, maskSize.value); output.convertTo(output, CV_8U); - outputSocket.setValue(output); + outputSocket.flagChanged(); } private enum Type { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java index b6efd38fa8..c661a3e37c 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindBlobsOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -28,7 +29,7 @@ iconName = "find-blobs") public class FindBlobsOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint minAreaHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Min Area", 1); private final SocketHint> circularityHint = SocketHints.Inputs @@ -41,7 +42,7 @@ public class FindBlobsOperation implements Operation { .initialValueSupplier(BlobsReport::new) .build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket minAreaSocket; private final InputSocket> circularitySocket; private final InputSocket colorSocket; @@ -80,7 +81,7 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); final Number minArea = minAreaSocket.getValue().get(); final List circularity = circularitySocket.getValue().get(); final Boolean darkBlobs = colorSocket.getValue().get(); @@ -109,6 +110,6 @@ public void perform() { blobs.add(new BlobsReport.Blob(keyPoint.pt().x(), keyPoint.pt().y(), keyPoint.size())); } - outputSocket.setValue(new BlobsReport(input, blobs)); + outputSocket.setValue(new BlobsReport(inputSocket.getValue().get(), blobs)); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java index 45b8506bdf..18285c16f1 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindContoursOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,8 +31,7 @@ iconName = "find-contours") public class FindContoursOperation implements Operation { - private final SocketHint inputHint = - new SocketHint.Builder<>(Mat.class).identifier("Input").build(); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint externalHint = SocketHints.createBooleanSocketHint("External Only", false); @@ -41,7 +41,7 @@ public class FindContoursOperation implements Operation { .identifier("Contours").initialValueSupplier(ContoursReport::new).build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket externalSocket; private final OutputSocket contoursSocket; @@ -73,7 +73,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); if (input.empty()) { return; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java index 32dc98932f..4f9597ce63 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/FindLinesOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,12 +31,12 @@ iconName = "find-lines") public class FindLinesOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint linesHint = new SocketHint.Builder<>(LinesReport.class) .identifier("Lines").initialValueSupplier(LinesReport::new).build(); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final OutputSocket linesReportSocket; @@ -63,17 +64,17 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final MatWrapper input = inputSocket.getValue().get(); final LineSegmentDetector lsd = linesReportSocket.getValue().get().getLineSegmentDetector(); final Mat lines = new Mat(); if (input.channels() == 1) { - lsd.detect(input, lines); + lsd.detect(input.getCpu(), lines); } else { // The line detector works on a single channel. If the input is a color image, we can just // give the line detector a grayscale version of it final Mat tmp = new Mat(); - cvtColor(input, tmp, COLOR_BGR2GRAY); + cvtColor(input.getCpu(), tmp, COLOR_BGR2GRAY); lsd.detect(tmp, lines); tmp.release(); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java index 8bf6135d62..b7958754cb 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSLThresholdOperation.java @@ -3,6 +3,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -33,7 +34,7 @@ public class HSLThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(HSLThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> hueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Hue", 0.0, 180.0); private final SocketHint> saturationHint = SocketHints.Inputs @@ -41,14 +42,14 @@ public class HSLThresholdOperation extends ThresholdOperation { private final SocketHint> luminanceHint = SocketHints.Inputs .createNumberListRangeSocketHint("Luminance", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> hueSocket; private final InputSocket> saturationSocket; private final InputSocket> luminanceSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -82,13 +83,13 @@ public List getOutputSockets() { @Override @SuppressWarnings("unchecked") public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("HSL Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = hueSocket.getValue().get(); final List channel2 = saturationSocket.getValue().get(); final List channel3 = luminanceSocket.getValue().get(); @@ -111,7 +112,7 @@ public void perform() { try { cvtColor(input, hls, COLOR_BGR2HLS); inRange(hls, low, high, output); - outputSocket.setValue(output); + outputSocket.flagChanged(); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java index 839acbd632..5b0ef05d11 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/HSVThresholdOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -33,7 +34,7 @@ public class HSVThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(HSVThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> hueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Hue", 0.0, 180.0); private final SocketHint> saturationHint = SocketHints.Inputs @@ -41,14 +42,14 @@ public class HSVThresholdOperation extends ThresholdOperation { private final SocketHint> valueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Value", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> hueSocket; private final InputSocket> saturationSocket; private final InputSocket> valueSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -82,13 +83,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("HSV Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = hueSocket.getValue().get(); final List channel2 = saturationSocket.getValue().get(); final List channel3 = valueSocket.getValue().get(); @@ -109,7 +110,7 @@ public void perform() { try { cvtColor(input, hsv, COLOR_BGR2HSV); inRange(hsv, low, high, output); - outputSocket.setValue(output); + outputSocket.flagChanged(); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java index f878a26d8d..1bc34e855f 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/LinesReport.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; @@ -9,7 +10,6 @@ import java.util.Collections; import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgproc.LineSegmentDetector; import static org.bytedeco.javacpp.opencv_imgproc.createLineSegmentDetector; @@ -24,7 +24,7 @@ @NoSocketTypeLabel public class LinesReport implements Publishable { private final LineSegmentDetector lsd; - private final Mat input; + private final MatWrapper input; private final List lines; /** @@ -32,7 +32,7 @@ public class LinesReport implements Publishable { * LinesReports. */ public LinesReport() { - this(createLineSegmentDetector(), new Mat(), Collections.emptyList()); + this(createLineSegmentDetector(), MatWrapper.emptyWrapper(), Collections.emptyList()); } /** @@ -40,7 +40,7 @@ public LinesReport() { * @param input The input matrix. * @param lines The lines that have been found. */ - public LinesReport(LineSegmentDetector lsd, Mat input, List lines) { + public LinesReport(LineSegmentDetector lsd, MatWrapper input, List lines) { this.lsd = lsd; this.input = input; this.lines = lines; @@ -53,7 +53,7 @@ protected LineSegmentDetector getLineSegmentDetector() { /** * @return The original image that the line detection was performed on. */ - public Mat getInput() { + public MatWrapper getInput() { return this.input; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java index 569ec1f953..4ce0599b5a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/MaskOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -25,16 +26,16 @@ iconName = "mask") public class MaskOperation implements Operation { - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); - private final SocketHint maskHint = SocketHints.Inputs.createMatSocketHint("Mask", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); + private final SocketHint maskHint = SocketHints.createImageSocketHint("Mask"); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; - private final InputSocket maskSocket; + private final InputSocket inputSocket; + private final InputSocket maskSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -63,14 +64,14 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); - final Mat mask = maskSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); + final Mat mask = maskSocket.getValue().get().getCpu(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); // Clear the output to black, then copy the input to it with the mask bitwise_xor(output, output, output); input.copyTo(output, mask); - outputSocket.setValue(output); + outputSocket.flagChanged(); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java index 52b331932a..6d570e4d9c 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/NormalizeOperation.java @@ -2,7 +2,9 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; +import edu.wpi.grip.core.operations.CudaOperation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -11,14 +13,15 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.opencv_core; +import org.bytedeco.javacpp.opencv_cudaarithm; + import java.util.List; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_core.NORM_INF; import static org.bytedeco.javacpp.opencv_core.NORM_L1; import static org.bytedeco.javacpp.opencv_core.NORM_L2; import static org.bytedeco.javacpp.opencv_core.NORM_MINMAX; -import static org.bytedeco.javacpp.opencv_core.normalize; /** * GRIP {@link Operation} for {@link org.bytedeco.javacpp.opencv_core#normalize}. @@ -27,40 +30,35 @@ summary = "Normalizes or remaps the values of pixels in an image", category = OperationCategory.IMAGE_PROCESSING, iconName = "opencv") -public class NormalizeOperation implements Operation { +public class NormalizeOperation extends CudaOperation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); private final SocketHint typeHint = SocketHints.createEnumSocketHint("Type", Type.MINMAX); private final SocketHint aHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Alpha", 0.0, 0, Double.MAX_VALUE); private final SocketHint bHint = SocketHints.Inputs .createNumberSpinnerSocketHint("Beta", 255, 0, Double.MAX_VALUE); - private final SocketHint dstHint = SocketHints.Inputs.createMatSocketHint("Output", true); - private final InputSocket srcSocket; private final InputSocket typeSocket; private final InputSocket alphaSocket; private final InputSocket betaSocket; - private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") public NormalizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.srcSocket = inputSocketFactory.create(srcHint); + super(inputSocketFactory, outputSocketFactory); this.typeSocket = inputSocketFactory.create(typeHint); this.alphaSocket = inputSocketFactory.create(aHint); this.betaSocket = inputSocketFactory.create(bHint); - - this.outputSocket = outputSocketFactory.create(dstHint); } @Override public List getInputSockets() { return ImmutableList.of( - srcSocket, + inputSocket, typeSocket, alphaSocket, - betaSocket + betaSocket, + gpuSocket ); } @@ -73,16 +71,21 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); - final Type type = typeSocket.getValue().get(); - final Number a = alphaSocket.getValue().get(); - final Number b = betaSocket.getValue().get(); - - final Mat output = outputSocket.getValue().get(); - - normalize(input, output, a.doubleValue(), b.doubleValue(), type.value, -1, null); + final MatWrapper input = inputSocket.getValue().get(); + final int type = typeSocket.getValue().get().value; + final double a = alphaSocket.getValue().get().doubleValue(); + final double b = betaSocket.getValue().get().doubleValue(); + + final MatWrapper output = outputSocket.getValue().get(); + + if (preferCuda() && input.channels() == 1) { + // CUDA normalize only works on single-channel images + opencv_cudaarithm.normalize(input.getGpu(), output.rawGpu(), a, b, type, -1); + } else { + opencv_core.normalize(input.getCpu(), output.rawCpu(), a, b, type, -1, null); + } - outputSocket.setValue(output); + outputSocket.flagChanged(); } private enum Type { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java index 0cf13c9282..2131766fe9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -24,7 +25,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgcodecs.CV_IMWRITE_JPEG_QUALITY; import static org.bytedeco.javacpp.opencv_imgcodecs.imencode; @@ -51,7 +51,7 @@ public class PublishVideoOperation implements Operation { private final Object imageLock = new Object(); private final BytePointer imagePointer = new BytePointer(); private final Thread serverThread; - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket qualitySocket; @SuppressWarnings("PMD.SingularField") private volatile boolean connected = false; @@ -99,9 +99,9 @@ public class PublishVideoOperation implements Operation { } // Copy the image data into a pre-allocated buffer, growing it if necessary - bufferSize = imagePointer.limit(); + bufferSize = (int) imagePointer.limit(); if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; + buffer = new byte[(int) imagePointer.limit()]; } imagePointer.get(buffer, 0, bufferSize); hasImage = false; @@ -144,8 +144,7 @@ public PublishVideoOperation(InputSocket.Factory inputSocketFactory) { if (numSteps != 0) { throw new IllegalStateException("Only one instance of PublishVideoOperation may exist"); } - this.inputSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("Image", - false)); + this.inputSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("Image")); this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSliderSocketHint("Quality", 80, 0, 100)); numSteps++; @@ -179,7 +178,7 @@ public void perform() { } synchronized (imageLock) { - imencode(".jpeg", inputSocket.getValue().get(), imagePointer, + imencode(".jpeg", inputSocket.getValue().get().getCpu(), imagePointer, new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); hasImage = true; imageLock.notifyAll(); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java index 481327ebf8..43ecb57dd9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/RGBThresholdOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -30,7 +31,7 @@ public class RGBThresholdOperation extends ThresholdOperation { private static final Logger logger = Logger.getLogger(RGBThresholdOperation.class.getName()); - private final SocketHint inputHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint = SocketHints.createImageSocketHint("Input"); private final SocketHint> redHint = SocketHints.Inputs .createNumberListRangeSocketHint("Red", 0.0, 255.0); private final SocketHint> greenHint = SocketHints.Inputs @@ -38,15 +39,15 @@ public class RGBThresholdOperation extends ThresholdOperation { private final SocketHint> blueHint = SocketHints.Inputs .createNumberListRangeSocketHint("Blue", 0.0, 255.0); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket> redSocket; private final InputSocket> greenSocket; private final InputSocket> blueSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") @@ -79,13 +80,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); if (input.channels() != 3) { throw new IllegalArgumentException("RGB Threshold needs a 3-channel input"); } - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); final List channel1 = redSocket.getValue().get(); final List channel2 = greenSocket.getValue().get(); final List channel3 = blueSocket.getValue().get(); @@ -106,7 +107,7 @@ public void perform() { try { inRange(input, low, high, output); - outputSocket.setValue(output); + outputSocket.flagChanged(); } catch (RuntimeException e) { logger.log(Level.WARNING, e.getMessage(), e); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java index 0b33581057..1ed7405e10 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java @@ -1,13 +1,13 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.PublishableObject; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.PublishValue; import edu.wpi.grip.core.operations.network.Publishable; import edu.wpi.grip.core.sockets.NoSocketTypeLabel; import com.google.common.collect.ImmutableList; -import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Rect; import java.util.ArrayList; @@ -20,12 +20,13 @@ @NoSocketTypeLabel public class RectsReport implements Publishable { - private final Mat image; + private final MatWrapper image; private final List rectangles; - public static final RectsReport NIL = new RectsReport(new Mat(), new ArrayList<>()); + public static final RectsReport NIL + = new RectsReport(MatWrapper.emptyWrapper(), new ArrayList<>()); - public RectsReport(Mat image, List rectangles) { + public RectsReport(MatWrapper image, List rectangles) { this.image = image; this.rectangles = ImmutableList.copyOf(rectangles); } @@ -33,7 +34,7 @@ public RectsReport(Mat image, List rectangles) { /** * Gets the image the rectangles are for. */ - public Mat getImage() { + public MatWrapper getImage() { return image; } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java index 20ea8817a3..45b59f9daa 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ResizeOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -32,19 +33,18 @@ iconName = "resize") public class ResizeOperation implements Operation { - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket widthSocket; private final InputSocket heightSocket; private final InputSocket interpolationSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; @Inject @SuppressWarnings("JavadocMethod") public ResizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - this.inputSocket = inputSocketFactory.create(SocketHints.Inputs - .createMatSocketHint("Input", false)); + this.inputSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("Input")); this.widthSocket = inputSocketFactory.create(SocketHints.Inputs .createNumberSpinnerSocketHint("Width", 640)); this.heightSocket = inputSocketFactory.create(SocketHints.Inputs @@ -52,8 +52,7 @@ public ResizeOperation(InputSocket.Factory inputSocketFactory, OutputSocket.Fact this.interpolationSocket = inputSocketFactory .create(SocketHints.createEnumSocketHint("Interpolation", Interpolation.CUBIC)); - this.outputSocket = outputSocketFactory.create(SocketHints.Outputs - .createMatSocketHint("Output")); + this.outputSocket = outputSocketFactory.create(SocketHints.createImageSocketHint("Output")); } @Override @@ -75,17 +74,17 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = inputSocket.getValue().get(); + final Mat input = inputSocket.getValue().get().getCpu(); final Number width = widthSocket.getValue().get(); final Number height = heightSocket.getValue().get(); final Interpolation interpolation = interpolationSocket.getValue().get(); - final Mat output = outputSocket.getValue().get(); + final Mat output = outputSocket.getValue().get().rawCpu(); resize(input, output, new Size(width.intValue(), height.intValue()), 0.0, 0.0, interpolation .value); - outputSocket.setValue(output); + outputSocket.flagChanged(); } private enum Interpolation { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java index 6f6fc08e60..90b2f9f130 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.core.FileManager; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -21,7 +22,6 @@ import java.util.Locale; import java.util.concurrent.TimeUnit; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_imgcodecs.CV_IMWRITE_JPEG_QUALITY; import static org.bytedeco.javacpp.opencv_imgcodecs.imencode; @@ -33,8 +33,8 @@ iconName = "publish-video") public class SaveImageOperation implements Operation { - private final SocketHint inputHint - = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint inputHint + = SocketHints.createImageSocketHint("Input"); private final SocketHint fileTypeHint = SocketHints.createEnumSocketHint("File type", FileTypes.JPEG); private final SocketHint qualityHint @@ -44,15 +44,15 @@ public class SaveImageOperation implements Operation { private final SocketHint activeHint = SocketHints.Inputs.createCheckboxSocketHint("Active", false); - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Output"); + private final SocketHint outputHint = SocketHints.createImageSocketHint("Output"); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final InputSocket fileTypesSocket; private final InputSocket qualitySocket; private final InputSocket periodSocket; private final InputSocket activeSocket; - private final OutputSocket outputSocket; + private final OutputSocket outputSocket; private final FileManager fileManager; private final BytePointer imagePointer = new BytePointer(); @@ -61,7 +61,8 @@ public class SaveImageOperation implements Operation { = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss-SSS"); private enum FileTypes { - JPEG, PNG; + JPEG, + PNG; @Override public String toString() { @@ -117,12 +118,14 @@ public void perform() { stopwatch.reset(); stopwatch.start(); - imencode("." + fileTypesSocket.getValue().get(), inputSocket.getValue().get(), imagePointer, + imencode("." + fileTypesSocket.getValue().get(), + inputSocket.getValue().get().getCpu(), + imagePointer, new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue())); byte[] buffer = new byte[128 * 1024]; - int bufferSize = imagePointer.limit(); + int bufferSize = (int) imagePointer.limit(); if (bufferSize > buffer.length) { - buffer = new byte[imagePointer.limit()]; + buffer = new byte[(int) imagePointer.limit()]; } imagePointer.get(buffer, 0, bufferSize); diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java index 5831b6864d..ce9d79d193 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/ThresholdMoving.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.composite; import edu.wpi.grip.annotation.operation.Description; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -23,16 +24,16 @@ + " previous and next image.") public class ThresholdMoving implements Operation { - private final InputSocket imageSocket; - private final OutputSocket outputSocket; + private final InputSocket imageSocket; + private final OutputSocket outputSocket; private final Mat lastImage; @Inject @SuppressWarnings("JavadocMethod") public ThresholdMoving(InputSocket.Factory inputSocketFactory, OutputSocket.Factory outputSocketFactory) { - imageSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("image", false)); - outputSocket = outputSocketFactory.create(SocketHints.Outputs.createMatSocketHint("moved")); + imageSocket = inputSocketFactory.create(SocketHints.createImageSocketHint("image")); + outputSocket = outputSocketFactory.create(SocketHints.createImageSocketHint("moved")); lastImage = new Mat(); } @@ -52,14 +53,14 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = imageSocket.getValue().get(); + final Mat input = imageSocket.getValue().get().getCpu(); final Size lastSize = lastImage.size(); final Size inputSize = input.size(); if (!lastImage.empty() && lastSize.height() == inputSize.height() && lastSize.width() == inputSize.width()) { - opencv_core.absdiff(input, lastImage, outputSocket.getValue().get()); + opencv_core.absdiff(input, lastImage, outputSocket.getValue().get().rawCpu()); } input.copyTo(lastImage); - outputSocket.setValue(outputSocket.getValue().get()); + outputSocket.flagChanged(); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java index 30a8f1a748..d84ebd835a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/composite/WatershedOperation.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -47,7 +48,7 @@ iconName = "opencv") public class WatershedOperation implements Operation { - private final SocketHint srcHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint srcHint = SocketHints.createImageSocketHint("Input"); private final SocketHint contoursHint = new SocketHint.Builder<>(ContoursReport.class) .identifier("Contours") @@ -60,7 +61,7 @@ public class WatershedOperation implements Operation { .initialValueSupplier(ContoursReport::new) .build(); - private final InputSocket srcSocket; + private final InputSocket srcSocket; private final InputSocket contoursSocket; private final OutputSocket outputSocket; @@ -100,7 +101,7 @@ public List getOutputSockets() { @Override public void perform() { - final Mat input = srcSocket.getValue().get(); + final Mat input = srcSocket.getValue().get().getCpu(); if (input.type() != CV_8UC3) { throw new IllegalArgumentException("Watershed only works on 8-bit, 3-channel images"); } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java b/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java index 4ffa01f351..ede2f88b09 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/network/networktables/NTManager.java @@ -30,6 +30,7 @@ import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; + import javax.inject.Inject; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java b/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java index 85a8f14757..0cda4e8179 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/network/ros/JavaToMessageConverter.java @@ -136,7 +136,7 @@ public void convert(J javaType, Message message, MessageFactory messageFactory) } private abstract static class SimpleConverter extends - JavaToMessageConverter { + JavaToMessageConverter { private final BiConsumer messageDataAssigner; private SimpleConverter(String type, BiConsumer messageDataAssigner) { diff --git a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java index 2b8ba3c2fa..47311e2fa3 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MatFieldAccessor.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -22,7 +23,7 @@ public class MatFieldAccessor implements CVOperation { private static final Mat defaultsMat = new Mat(); - private final SocketHint matHint = SocketHints.Inputs.createMatSocketHint("Input", false); + private final SocketHint matHint = SocketHints.createImageSocketHint("Input"); private final SocketHint sizeHint = SocketHints.Inputs.createSizeSocketHint("size", true); private final SocketHint emptyHint = SocketHints.Outputs .createBooleanSocketHint("empty", defaultsMat.empty()); @@ -36,7 +37,7 @@ public class MatFieldAccessor implements CVOperation { .createNumberSocketHint("high value", defaultsMat.highValue()); - private final InputSocket inputSocket; + private final InputSocket inputSocket; private final OutputSocket sizeSocket; private final OutputSocket emptySocket; @@ -80,13 +81,13 @@ public List getOutputSockets() { @Override public void perform() { - final Mat inputMat = inputSocket.getValue().get(); + MatWrapper wrapper = inputSocket.getValue().get(); - sizeSocket.setValue(inputMat.size()); - emptySocket.setValue(inputMat.empty()); - channelsSocket.setValue(inputMat.channels()); - colsSocket.setValue(inputMat.cols()); - rowsSocket.setValue(inputMat.rows()); - highValueSocket.setValue(inputMat.highValue()); + sizeSocket.setValue(wrapper.size()); + emptySocket.setValue(wrapper.empty()); + channelsSocket.setValue(wrapper.channels()); + colsSocket.setValue(wrapper.cols()); + rowsSocket.setValue(wrapper.rows()); + highValueSocket.setValue(wrapper.highValue()); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java index 84e364026a..5d975d46a1 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java @@ -2,6 +2,7 @@ import edu.wpi.grip.annotation.operation.Description; import edu.wpi.grip.annotation.operation.OperationCategory; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -10,6 +11,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.opencv_core; import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Point; @@ -25,10 +27,8 @@ iconName = "opencv") public class MinMaxLoc implements CVOperation { - private final SocketHint srcInputHint = SocketHints.Inputs - .createMatSocketHint("Image", false); - private final SocketHint maskInputHint = SocketHints.Inputs - .createMatSocketHint("Mask", true); + private final SocketHint srcInputHint = SocketHints.createImageSocketHint("Image"); + private final SocketHint maskInputHint = SocketHints.createImageSocketHint("Mask"); private final SocketHint minValOutputHint = SocketHints.Outputs .createNumberSocketHint("Min Val", 0); @@ -40,8 +40,8 @@ public class MinMaxLoc implements CVOperation { private final SocketHint maxLocOutputHint = SocketHints.Outputs .createPointSocketHint("Max Loc"); - private final InputSocket srcSocket; - private final InputSocket maskSocket; + private final InputSocket srcSocket; + private final InputSocket maskSocket; private final OutputSocket minValSocket; private final OutputSocket maxValSocket; @@ -81,20 +81,20 @@ public List getOutputSockets() { @Override public void perform() { - final Mat src = srcSocket.getValue().get(); - Mat mask = maskSocket.getValue().get(); + final Mat src = srcSocket.getValue().get().getCpu(); + Mat mask = maskSocket.getValue().get().getCpu(); if (mask.empty()) { mask = null; } - final double[] minVal = new double[1]; - final double[] maxVal = new double[1]; + DoublePointer minVal = new DoublePointer(0.0); + DoublePointer maxVal = new DoublePointer(0.0); final Point minLoc = minLocSocket.getValue().get(); final Point maxLoc = maxLocSocket.getValue().get(); opencv_core.minMaxLoc(src, minVal, maxVal, minLoc, maxLoc, mask); - minValSocket.setValue(minVal[0]); - maxValSocket.setValue(maxVal[0]); - minLocSocket.setValue(minLocSocket.getValue().get()); - maxLocSocket.setValue(maxLocSocket.getValue().get()); + minValSocket.setValue(minVal.get()); + maxValSocket.setValue(maxVal.get()); + minLocSocket.flagChanged(); + maxLocSocket.flagChanged(); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..4abb81ce7c --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationCudaOperation.java @@ -0,0 +1,79 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class FiveSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final InputSocket input5; + private final OutputSocket output; + private final Performer performer; + + FiveSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.input5 = isf.create(t5SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + input5, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + input5.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, T5 src5, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java index 80fea16669..bb2be4a3cc 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FiveSourceOneDestinationOperation.java @@ -59,7 +59,7 @@ public void perform() { input4.getValue().get(), input5.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..f69cbc4419 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationCudaOperation.java @@ -0,0 +1,74 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class FourSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final OutputSocket output; + private final Performer performer; + + FourSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java index de040f9e18..302a0a0079 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/FourSourceOneDestinationOperation.java @@ -72,7 +72,7 @@ public void perform() { input4.getValue().get(), output.getValue().get() ); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..68cc21728f --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationCudaOperation.java @@ -0,0 +1,55 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class OneSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final OutputSocket output; + private final Performer performer; + + OneSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform(input1.getValue().get(), preferCuda(), output.getValue().get()); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java index 826497f9c8..22cefe9844 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/OneSourceOneDestinationOperation.java @@ -40,7 +40,7 @@ public List getOutputSockets() { @SuppressWarnings("OptionalGetWithoutIsPresent") public void perform() { performer.perform(input1.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..b7429dd41e --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationCudaOperation.java @@ -0,0 +1,91 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class SevenSourceOneDestinationCudaOperation + extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final InputSocket input5; + private final InputSocket input6; + private final InputSocket input7; + private final OutputSocket output; + private final Performer performer; + + SevenSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint t7SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.input5 = isf.create(t5SocketHint); + this.input6 = isf.create(t6SocketHint); + this.input7 = isf.create(t7SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + input5, + input6, + input7, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + input5.getValue().get(), + input6.getValue().get(), + input7.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, T5 src5, T6 src6, T7 src7, + boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java index 017e41bbaf..650ee93884 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SevenSourceOneDestinationOperation.java @@ -69,7 +69,7 @@ public void perform() { input7.getValue().get(), output.getValue().get() ); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..a426027a43 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationCudaOperation.java @@ -0,0 +1,84 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class SixSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final InputSocket input4; + private final InputSocket input5; + private final InputSocket input6; + private final OutputSocket output; + private final Performer performer; + + SixSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.input4 = isf.create(t4SocketHint); + this.input5 = isf.create(t5SocketHint); + this.input6 = isf.create(t6SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + input4, + input5, + input6, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + input4.getValue().get(), + input5.getValue().get(), + input6.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, T4 src4, T5 src5, T6 src6, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java index 6c1ece513b..4c03bd1c3e 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/SixSourceOneDestinationOperation.java @@ -65,7 +65,7 @@ public void perform() { input6.getValue().get(), output.getValue().get() ); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java index 415e248d1d..31a4862bd9 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/TemplateFactory.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.templated; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Operation; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -9,8 +10,6 @@ import com.google.inject.Singleton; -import org.bytedeco.javacpp.opencv_core.Mat; - import java.util.function.Supplier; /** @@ -113,38 +112,137 @@ public Supplier create( performer); } + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint rSocketHint, + OneSourceOneDestinationCudaOperation.Performer performer) { + return () -> new OneSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, + rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint rSocketHint, + TwoSourceOneDestinationCudaOperation.Performer performer) { + return () -> new TwoSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint rSocketHint, + ThreeSourceOneDestinationCudaOperation.Performer performer) { + return () -> new ThreeSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint rSocketHint, + FourSourceOneDestinationCudaOperation.Performer performer) { + return () -> new FourSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint rSocketHint, + FiveSourceOneDestinationCudaOperation.Performer performer) { + return () -> new FiveSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, t5SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint rSocketHint, + SixSourceOneDestinationCudaOperation.Performer performer) { + return () -> new SixSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, t5SocketHint, t6SocketHint, rSocketHint, performer); + } + + public Supplier createCuda( + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint t4SocketHint, + SocketHint t5SocketHint, + SocketHint t6SocketHint, + SocketHint t7SocketHint, + SocketHint rSocketHint, + SevenSourceOneDestinationCudaOperation.Performer performer) { + return () -> new SevenSourceOneDestinationCudaOperation<>(isf, osf, t1SocketHint, t2SocketHint, + t3SocketHint, t4SocketHint, t5SocketHint, t6SocketHint, t7SocketHint, rSocketHint, + performer); + } + public Supplier createAllMatTwoSource( - SocketHint matSocketHint, - SocketHint matSocketHint2, - SocketHint matSocketHint3, - TwoSourceOneDestinationOperation.Performer performer) { + SocketHint matSocketHint, + SocketHint matSocketHint2, + SocketHint matSocketHint3, + TwoSourceOneDestinationOperation.Performer performer) { return create(matSocketHint, matSocketHint2, matSocketHint3, performer); } - public Supplier createAllMatTwoSource(TwoSourceOneDestinationOperation - .Performer performer) { - return createAllMatTwoSource(srcSocketHint(Mat.class, 1), srcSocketHint(Mat.class, 2), + public Supplier createAllMatTwoSource( + TwoSourceOneDestinationOperation.Performer performer) { + return createAllMatTwoSource(srcSocketHint(MatWrapper.class, 1), + srcSocketHint(MatWrapper.class, 2), dstMatSocketHint(), performer); } + public Supplier createAllMatTwoSourceCuda( + TwoSourceOneDestinationCudaOperation.Performer + performer) { + return createCuda( + srcSocketHint(MatWrapper.class, 1), + srcSocketHint(MatWrapper.class, 2), + dstMatSocketHint(), + performer + ); + } + public Supplier createAllMatOneSource( - SocketHint matSocketHint, - SocketHint matSocketHint2, - OneSourceOneDestinationOperation.Performer performer) { + SocketHint matSocketHint, + SocketHint matSocketHint2, + OneSourceOneDestinationOperation.Performer performer) { return create(matSocketHint, matSocketHint2, performer); } - public Supplier createAllMatOneSource(OneSourceOneDestinationOperation - .Performer performer) { - return createAllMatOneSource(srcSocketHint(Mat.class, 1), dstMatSocketHint(), performer); + public Supplier createAllMatOneSource( + OneSourceOneDestinationOperation.Performer performer) { + return createAllMatOneSource(srcSocketHint(MatWrapper.class, 1), dstMatSocketHint(), + performer); } + public Supplier createAllMatOneSourceCuda( + OneSourceOneDestinationCudaOperation.Performer performer) { + return createCuda( + srcSocketHint(MatWrapper.class, 1), + dstMatSocketHint(), + performer + ); + } private SocketHint srcSocketHint(Class srcType, int index) { return new SocketHint.Builder<>(srcType).identifier("src" + index).build(); } - private SocketHint dstMatSocketHint() { - return SocketHints.Outputs.createMatSocketHint("dst"); + private SocketHint dstMatSocketHint() { + return SocketHints.createImageSocketHint("dst"); } } diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..bee043b757 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationCudaOperation.java @@ -0,0 +1,69 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class ThreeSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final InputSocket input3; + private final OutputSocket output; + private final Performer performer; + + ThreeSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint t3SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.input3 = isf.create(t3SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + input3, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + input3.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, T3 src3, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java index 071ab60518..8556934f2a 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/ThreeSourceOneDestinationOperation.java @@ -46,7 +46,7 @@ public List getOutputSockets() { public void perform() { performer.perform(input1.getValue().get(), input2.getValue().get(), input3.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationCudaOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationCudaOperation.java new file mode 100644 index 0000000000..02381f100b --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationCudaOperation.java @@ -0,0 +1,64 @@ +package edu.wpi.grip.core.operations.templated; + +import edu.wpi.grip.core.operations.CudaOperation; +import edu.wpi.grip.core.sockets.InputSocket; +import edu.wpi.grip.core.sockets.OutputSocket; +import edu.wpi.grip.core.sockets.SocketHint; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +final class TwoSourceOneDestinationCudaOperation extends CudaOperation { + + private final InputSocket input1; + private final InputSocket input2; + private final OutputSocket output; + private final Performer performer; + + TwoSourceOneDestinationCudaOperation(InputSocket.Factory isf, + OutputSocket.Factory osf, + SocketHint t1SocketHint, + SocketHint t2SocketHint, + SocketHint rSocketHint, + Performer performer) { + super(isf, osf); + this.input1 = isf.create(t1SocketHint); + this.input2 = isf.create(t2SocketHint); + this.output = osf.create(rSocketHint); + this.performer = performer; + } + + @Override + public List getInputSockets() { + return ImmutableList.of( + input1, + input2, + gpuSocket + ); + } + + @Override + public List getOutputSockets() { + return ImmutableList.of( + output + ); + } + + @Override + public void perform() { + performer.perform( + input1.getValue().get(), + input2.getValue().get(), + preferCuda(), + output.getValue().get() + ); + output.flagChanged(); + } + + @FunctionalInterface + public interface Performer { + void perform(T1 src1, T2 src2, boolean preferCuda, R dst); + } + +} diff --git a/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java index 874aab3d0f..39802834a8 100644 --- a/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java +++ b/core/src/main/java/edu/wpi/grip/core/operations/templated/TwoSourceOneDestinationOperation.java @@ -44,7 +44,7 @@ public List getOutputSockets() { @SuppressWarnings("OptionalGetWithoutIsPresent") public void perform() { performer.perform(input1.getValue().get(), input2.getValue().get(), output.getValue().get()); - output.setValue(output.getValue().get()); + output.flagChanged(); } @FunctionalInterface diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/CudaSocket.java b/core/src/main/java/edu/wpi/grip/core/sockets/CudaSocket.java new file mode 100644 index 0000000000..d28d6c6099 --- /dev/null +++ b/core/src/main/java/edu/wpi/grip/core/sockets/CudaSocket.java @@ -0,0 +1,62 @@ +package edu.wpi.grip.core.sockets; + +import edu.wpi.grip.core.cuda.CudaDetector; + +import com.google.common.eventbus.EventBus; + +import java.util.Objects; +import java.util.Optional; + +import javax.annotation.Nullable; + +/** + * A type of input socket that lets an operation know that it should prefer to use a + * CUDA-accelerated code path. If no compatible CUDA runtime is available, sockets of this type + * will always have a value of {@code false} and cannot be changed. + */ +public class CudaSocket extends InputSocketImpl { + + private static final Optional NO = Optional.of(false); + + private final boolean isCudaAvailable; + + CudaSocket(EventBus eventBus, CudaDetector detector, SocketHint socketHint) { + super(eventBus, socketHint); + + // Cache the value, since it's not likely that the user will install or uninstall a CUDA + // runtime while the app is running + isCudaAvailable = detector.isCompatibleCudaInstalled(); + } + + @Override + public Optional getValue() { + if (isCudaAvailable) { + return super.getValue(); + } else { + return NO; + } + } + + @Override + public void setValue(@Nullable Boolean value) { + if (isCudaAvailable) { + super.setValue(value); + } else { + super.setValue(false); + } + } + + @Override + public void setValueOptional(Optional optionalValue) { + Objects.requireNonNull(optionalValue, "optionalValue"); + if (isCudaAvailable) { + super.setValueOptional(optionalValue); + } else { + super.setValueOptional(NO); + } + } + + public boolean isCudaAvailable() { + return isCudaAvailable; + } +} diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java index b582c84986..15ed50e5c2 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocket.java @@ -30,6 +30,8 @@ public interface InputSocket extends Socket { interface Factory { InputSocket create(SocketHint hint); + + CudaSocket createCuda(SocketHint hint); } /** diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java index 40456510e5..32b450ad41 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/InputSocketImpl.java @@ -2,6 +2,7 @@ import edu.wpi.grip.core.Connection; +import edu.wpi.grip.core.cuda.CudaDetector; import com.google.common.eventbus.EventBus; import com.google.inject.Inject; @@ -56,16 +57,23 @@ public void onValueChanged() { @Singleton public static class FactoryImpl implements Factory { private final EventBus eventBus; + private final CudaDetector cudaDetector; @Inject - FactoryImpl(EventBus eventBus) { + FactoryImpl(EventBus eventBus, CudaDetector cudaDetector) { this.eventBus = eventBus; + this.cudaDetector = cudaDetector; } @Override public InputSocket create(SocketHint hint) { return new InputSocketImpl<>(eventBus, hint); } + + @Override + public CudaSocket createCuda(SocketHint hint) { + return new CudaSocket(eventBus, cudaDetector, hint); + } } diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java b/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java index c8cdcfd038..70866617b1 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/Socket.java @@ -41,6 +41,15 @@ default void onValueChanged() { /* no-op */ } + /** + * Notifies this socket that the value changed. This is usually only needed for sockets that + * contain mutable data such as images or other native classes (Point, Size, etc) that are + * written to by OpenCV operations. + */ + default void flagChanged() { + setValueOptional(getValue()); + } + /** * @return The value currently stored in this socket. */ diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java index 0341102b28..b0005e73df 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java @@ -1,5 +1,7 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.base.MoreObjects; import org.bytedeco.javacpp.opencv_objdetect.CascadeClassifier; @@ -121,7 +123,7 @@ public String getTypeLabel() { || type.equals(List.class)) { // Enums labels are kind of redundant, and Lists actually represent ranges return ""; - } else if (Mat.class.equals(type)) { + } else if (Mat.class.equals(type) || MatWrapper.class.equals(type)) { // "Mats" represent images return "Image"; } else if (CascadeClassifier.class.equals(type)) { diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java index 6fbb8d1e32..50924e6876 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketHints.java @@ -1,9 +1,10 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.reflect.TypeToken; -import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_core.Point; import org.bytedeco.javacpp.opencv_core.Size; @@ -23,6 +24,13 @@ public final class SocketHints { private SocketHints() { /* no op */ } + public static SocketHint createImageSocketHint(String identifier) { + return new SocketHint.Builder<>(MatWrapper.class) + .identifier(identifier) + .initialValueSupplier(MatWrapper::emptyWrapper) + .build(); + } + @SuppressWarnings("unchecked") public static > SocketHint createEnumSocketHint(final String identifier, final T defaultValue) { @@ -93,11 +101,6 @@ private static SocketHint.Builder createNumberSocketHintBuilder(final St public static final class Inputs { private Inputs() { /* no op */ } - public static SocketHint createMatSocketHint(final String identifier, - final boolean withDefault) { - return createObjectSocketHintBuilder(identifier, Mat.class, Mat::new, withDefault).build(); - } - public static SocketHint createSizeSocketHint(final String identifier, final boolean withDefault) { return createObjectSocketHintBuilder(identifier, Size.class, Size::new, withDefault).build(); @@ -170,10 +173,6 @@ public static SocketHint createCheckboxSocketHint( public static final class Outputs { private Outputs() { /* no op */ } - public static SocketHint createMatSocketHint(final String identifier) { - return Inputs.createMatSocketHint(identifier, true); - } - public static SocketHint createPointSocketHint(final String identifier) { return Inputs.createPointSocketHint(identifier, true); } diff --git a/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java b/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java index 84b902b727..2e4e437b7f 100644 --- a/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java +++ b/core/src/main/java/edu/wpi/grip/core/sockets/SocketImpl.java @@ -62,6 +62,12 @@ public void setValueOptional(Optional optionalValue) { eventBus.post(new SocketChangedEvent(this)); } + @Override + public void flagChanged() { + onValueChanged(); + eventBus.post(new SocketChangedEvent(this)); + } + @Override public Optional getValue() { if (!this.value.isPresent()) { diff --git a/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java b/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java index 3e3a95bba6..f22da6a2df 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -79,11 +80,10 @@ public class CameraSource extends Source implements RestartableService { private final Properties properties; - private final SocketHint imageOutputHint = SocketHints.Inputs.createMatSocketHint("Image", - true); + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); private final SocketHint frameRateOutputHint = SocketHints.createNumberSocketHint("Frame Rate", 0); - private final OutputSocket frameOutputSocket; + private final OutputSocket frameOutputSocket; private final OutputSocket frameRateOutputSocket; private final Supplier grabberSupplier; private final AtomicBoolean isNewFrame = new AtomicBoolean(false); @@ -240,9 +240,9 @@ protected boolean updateOutputSockets() { // The camera frame thread should not try to modify the transfer mat while it is being // written to the pipeline synchronized (currentFrameTransferMat) { - currentFrameTransferMat.copyTo(frameOutputSocket.getValue().get()); + frameOutputSocket.getValue().ifPresent(m -> m.set(currentFrameTransferMat)); } - frameOutputSocket.setValueOptional(frameOutputSocket.getValue()); + frameOutputSocket.flagChanged(); // Update the frame rate value frameRateOutputSocket.setValue(frameRate); diff --git a/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java b/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java index 3c36260cd5..c9663dc70a 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/HttpSource.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -19,7 +20,6 @@ import com.thoughtworks.xstream.annotations.XStreamAlias; import org.bytedeco.javacpp.opencv_core.Mat; -import org.bytedeco.javacpp.opencv_imgcodecs; import java.util.HashMap; import java.util.List; @@ -27,6 +27,9 @@ import java.util.Properties; import java.util.function.Consumer; +import static org.bytedeco.javacpp.opencv_imgcodecs.CV_LOAD_IMAGE_COLOR; +import static org.bytedeco.javacpp.opencv_imgcodecs.imdecode; + /** * Provides a way to generate a {@link Mat Mat} from an image that has been POSTed to the * internal HTTP server. @@ -50,9 +53,9 @@ public class HttpSource extends Source { */ private final HttpImageHandler imageHandler; - private final OutputSocket imageOutput; - private final SocketHint outputHint = SocketHints.Outputs.createMatSocketHint("Image"); - private final Mat image = new Mat(); + private final OutputSocket imageOutput; + private final SocketHint outputHint = SocketHints.createImageSocketHint("Image"); + private final MatWrapper image = MatWrapper.emptyWrapper(); private final Consumer callback; private final EventBus eventBus; private String path; @@ -99,7 +102,7 @@ public interface Factory { } private void setImage(Mat image) { - image.copyTo(this.image); + this.image.set(image); eventBus.post(new SourceHasPendingUpdateEvent(this)); } @@ -121,7 +124,8 @@ protected boolean updateOutputSockets() { // No data, don't bother converting return false; } - imageOutput.setValue(opencv_imgcodecs.imdecode(image, opencv_imgcodecs.CV_LOAD_IMAGE_COLOR)); + imageOutput.getValue().get().set(imdecode(image.getCpu(), CV_LOAD_IMAGE_COLOR)); + imageOutput.flagChanged(); return true; } diff --git a/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java index 94fd3989ec..db3808d5f4 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/ImageFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -36,8 +37,8 @@ public final class ImageFileSource extends Source { private final String name; private final String path; - private final SocketHint imageOutputHint = SocketHints.Outputs.createMatSocketHint("Image"); - private final OutputSocket outputSocket; + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); + private final OutputSocket outputSocket; /** * @param exceptionWitnessFactory Factory to create the exceptionWitness @@ -116,8 +117,8 @@ private void loadImage(String path) throws IOException { } private void loadImage(String path, final int flags) throws IOException { - ImageLoadingUtility.loadImage(path, flags, this.outputSocket.getValue().get()); - this.outputSocket.setValue(this.outputSocket.getValue().get()); + ImageLoadingUtility.loadImage(path, flags, this.outputSocket.getValue().get().rawCpu()); + this.outputSocket.flagChanged(); } diff --git a/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java index 0c51b9cc2e..9553f60cd3 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/MultiImageFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.PreviousNext; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; @@ -38,9 +39,8 @@ public final class MultiImageFileSource extends Source implements PreviousNext { private static final String INDEX_PROPERTY = "index"; private static final String SIZE_PROPERTY = "numImages"; - private final SocketHint imageOutputHint = SocketHints.Inputs.createMatSocketHint("Image", - true); - private final OutputSocket outputSocket; + private final SocketHint imageOutputHint = SocketHints.createImageSocketHint("Image"); + private final OutputSocket outputSocket; private final EventBus eventBus; private final List paths; @@ -116,7 +116,9 @@ private static String getPathProperty(int index) { * paths. * * @param paths The paths of all of the images. + * * @return The list of Mats loaded from the file system. + * * @throws IOException if one of the images fails to load */ private static Mat[] createImagesArray(List paths) throws IOException { @@ -167,8 +169,11 @@ protected List createOutputSockets() { @Override protected boolean updateOutputSockets() { - if (!currentImage.equals(outputSocket.getValue())) { - outputSocket.setValueOptional(currentImage); + if (outputSocket.getValue() + .map(m -> m.getCpu()) + .map(m -> !currentImage.get().equals(m)) + .orElse(false)) { + outputSocket.getValue().ifPresent(m -> currentImage.ifPresent(m::set)); return true; } else { return false; @@ -192,6 +197,7 @@ public Properties getProperties() { * remain within the bounds of the image array. * * @param delta the value to add to the index when getting the image + * * @return The matrix at the given index in the array. */ private Mat addIndexAndGetImageByOffset(final int delta) { diff --git a/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java b/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java index 0d6375ce66..4b7321aa0c 100644 --- a/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java +++ b/core/src/main/java/edu/wpi/grip/core/sources/VideoFileSource.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Source; import edu.wpi.grip.core.events.SourceHasPendingUpdateEvent; import edu.wpi.grip.core.events.SourceRemovedEvent; @@ -41,9 +42,9 @@ public class VideoFileSource extends Source implements Pausable { private final String path; - private final SocketHint imageHint = SocketHints.Outputs.createMatSocketHint("Image"); + private final SocketHint imageHint = SocketHints.createImageSocketHint("Image"); private final SocketHint fpsHint = SocketHints.Outputs.createNumberSocketHint("FPS", 0); - private final OutputSocket imageSocket; + private final OutputSocket imageSocket; private final OutputSocket fpsSocket; private final Mat workingMat = new Mat(); private final AtomicBoolean isNewFrame = new AtomicBoolean(false); @@ -102,7 +103,7 @@ protected boolean updateOutputSockets() { if (isNewFrame.compareAndSet(true, false)) { // New frame, update outputs synchronized (workingMat) { - workingMat.copyTo(imageSocket.getValue().get()); + workingMat.copyTo(imageSocket.getValue().get().rawCpu()); } imageSocket.setValue(imageSocket.getValue().get()); // force the socket to update return true; diff --git a/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java b/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java index 85755b5b7c..333df2cbf9 100644 --- a/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java +++ b/core/src/main/java/edu/wpi/grip/core/util/MetaInfReader.java @@ -31,7 +31,7 @@ public final class MetaInfReader { * @throws IOException if the file could not be read */ public static Stream> readClasses(String fileName) throws IOException { - return read(fileName) + return readLines(fileName) .map(MetaInfReader::classForNameOrNull) .map(c -> (Class) c) .filter(Objects::nonNull); @@ -45,15 +45,15 @@ public static Stream> readClasses(String fileName) throws IOExcepti * @throws IOException if no file exists with the given name, or if the file exists but cannot * be read * - * @see #read(InputStream) + * @see #readLines(InputStream) */ - private static Stream read(String fileName) throws IOException { + public static Stream readLines(String fileName) throws IOException { checkNotNull(fileName, "fileName"); InputStream stream = MetaInfReader.class.getResourceAsStream("/META-INF/" + fileName); if (stream == null) { throw new IOException("No resource /META-INF/" + fileName + " found"); } - return read(stream); + return readLines(stream); } /** @@ -68,7 +68,7 @@ private static Stream read(String fileName) throws IOException { * @throws IOException if the stream could not be read from or safely closed */ @VisibleForTesting - static Stream read(InputStream inputStream) throws IOException { + static Stream readLines(InputStream inputStream) throws IOException { List lines; try (InputStreamReader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); BufferedReader bufferedReader = new BufferedReader(reader)) { diff --git a/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java b/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java index 3aa326aeb5..d1b063e2b9 100644 --- a/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java +++ b/core/src/main/java/edu/wpi/grip/core/util/SafeShutdown.java @@ -14,6 +14,40 @@ public final class SafeShutdown { public static volatile boolean stopping = false; + /** + * Exit codes used by the GRIP application. + */ + public enum ExitCode { + /** + * Clean shutdown. + */ + SAFE_SHUTDOWN(0x00), + /** + * An unknown exception was thrown and uncaught. + */ + MISC_ERROR(0x01), + + /** + * The HTTP server cannot start (typically due to the port already being in use). + */ + HTTP_SERVER_COULD_NOT_START(0x02), + /** + * CUDA is required by OpenCV but no compatible runtime is available on the system. + */ + CUDA_UNAVAILABLE(0x04), + ; + + private final int code; + + ExitCode(int code) { + this.code = code; + } + + public int getCode() { + return code; + } + } + static { /* * Shutdown hook run order is non-deterministic but this increases our likelihood of @@ -40,7 +74,7 @@ public void run() { * has been flagged true. This is nullable. * @see System#exit(int) */ - public static void exit(int statusCode, @Nullable PreSystemExitHook hook) { + public static void exit(ExitCode statusCode, @Nullable PreSystemExitHook hook) { flagStopping(); try { if (hook != null) { @@ -48,7 +82,7 @@ public static void exit(int statusCode, @Nullable PreSystemExitHook hook) { } } finally { Logger.getLogger(SafeShutdown.class.getName()).info("Exiting GRIP"); - System.exit(statusCode); + System.exit(statusCode.getCode()); } } @@ -57,9 +91,9 @@ public static void exit(int statusCode, @Nullable PreSystemExitHook hook) { * Helper method that passes null as the PreSystemExitHook. * * @param statusCode exit status. - * @see #exit(int) + * @see #exit(ExitCode, PreSystemExitHook) */ - public static void exit(int statusCode) { + public static void exit(ExitCode statusCode) { exit(statusCode, null); } diff --git a/core/src/test/java/edu/wpi/grip/core/AddOperation.java b/core/src/test/java/edu/wpi/grip/core/AddOperation.java index e86c453294..7d1775ae69 100644 --- a/core/src/test/java/edu/wpi/grip/core/AddOperation.java +++ b/core/src/test/java/edu/wpi/grip/core/AddOperation.java @@ -11,7 +11,6 @@ import com.google.common.eventbus.EventBus; import org.bytedeco.javacpp.opencv_core; -import org.bytedeco.javacpp.opencv_core.Mat; import java.util.List; @@ -22,13 +21,13 @@ public class AddOperation implements Operation { public static final OperationDescription DESCRIPTION = OperationDescription .builder().name("OpenCV Add").summary("Compute the per-pixel sum of two images.").build(); - private final SocketHint aHint = SocketHints.Inputs.createMatSocketHint("a", false); - private final SocketHint bHint = SocketHints.Inputs.createMatSocketHint("b", false); - private final SocketHint sumHint = SocketHints.Inputs.createMatSocketHint("sum", true); + private final SocketHint aHint = SocketHints.createImageSocketHint("a"); + private final SocketHint bHint = SocketHints.createImageSocketHint("b"); + private final SocketHint sumHint = SocketHints.createImageSocketHint("sum"); - private InputSocket a; - private InputSocket b; - private OutputSocket sum; + private InputSocket a; + private InputSocket b; + private OutputSocket sum; public AddOperation(EventBus eventBus) { this(new MockInputSocketFactory(eventBus), new MockOutputSocketFactory(eventBus)); @@ -56,6 +55,8 @@ public List getOutputSockets() { @Override public void perform() { - opencv_core.add(a.getValue().get(), b.getValue().get(), sum.getValue().get()); + opencv_core.add(a.getValue().get().getCpu(), + b.getValue().get().getCpu(), + sum.getValue().get().rawCpu()); } } diff --git a/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java b/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java index fa4be39545..b52c95e84e 100644 --- a/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java +++ b/core/src/test/java/edu/wpi/grip/core/CoreSanityTest.java @@ -28,7 +28,8 @@ public CoreSanityTest() { ManualPipelineRunner.class, SubtractionOperation.class, Main.class, - CoreCommandLineHelper.class + CoreCommandLineHelper.class, + OperationDescription.class ).contains(c)); setDefault(OutputSocket.class, new MockOutputSocket("Mock Out")); setDefault(InputSocket.class, new MockInputSocket("Mock In")); diff --git a/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java b/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java index 49dd26f583..48ae07d2b6 100644 --- a/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java +++ b/core/src/test/java/edu/wpi/grip/core/PipelineRunnerTest.java @@ -19,6 +19,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import net.jodah.concurrentunit.Waiter; + import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/core/src/test/java/edu/wpi/grip/core/cuda/CudaVerifierTest.java b/core/src/test/java/edu/wpi/grip/core/cuda/CudaVerifierTest.java new file mode 100644 index 0000000000..020bc26c2e --- /dev/null +++ b/core/src/test/java/edu/wpi/grip/core/cuda/CudaVerifierTest.java @@ -0,0 +1,51 @@ +package edu.wpi.grip.core.cuda; + +import org.junit.Test; + +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class CudaVerifierTest { + + @Test + public void testVerifyNonCudaRuntime() { + CudaVerifier verifier = new NonExitingVerifier(() -> false, () -> false); + assertTrue("Not using CUDA should always pass verification", verifier.verify()); + } + + @Test + public void testUsingCudaWithNoAvailableRuntime() { + CudaVerifier verifier = new NonExitingVerifier(() -> true, () -> false); + assertFalse("Using CUDA but no available runtime should fail verification", verifier.verify()); + } + + @Test + public void testExitsWhenFailsVerification() { + NonExitingVerifier verifier = new NonExitingVerifier(() -> true, () -> false); + + verifier.verifyCuda(); + assertEquals(1, verifier.getExitCount()); + } + + private static final class NonExitingVerifier extends CudaVerifier { + + private int exitCount = 0; + + public NonExitingVerifier(AccelerationMode accelerationMode, CudaDetector cudaDetector) { + super(accelerationMode, cudaDetector, new Properties()); + } + + @Override + void exit() { + exitCount++; + } + + public int getExitCount() { + return exitCount; + } + } + +} diff --git a/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java b/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java index 8920b3cbdc..6923b5b802 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/OperationsUtil.java @@ -13,9 +13,14 @@ public class OperationsUtil { @Inject private CVOperations cvOperations; + @Inject + private Operations operations; public ImmutableList operations() { - return cvOperations.operations(); + return ImmutableList.builder() + .addAll(cvOperations.operations()) + .addAll(operations.operations()) + .build(); } public OperationMetaData getMetaData(String opName) { diff --git a/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java b/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java index 4946278eaf..3b89f0ccbd 100644 --- a/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java +++ b/core/src/test/java/edu/wpi/grip/core/operations/opencv/AddOperationTest.java @@ -1,6 +1,7 @@ package edu.wpi.grip.core.operations.opencv; import edu.wpi.grip.core.AddOperation; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; @@ -55,20 +56,20 @@ public void testAddMatrixOfOnesToMatrixOfTwosEqualsMatrixOfThrees() { // Given List inputs = addition.getInputSockets(); List outputs = addition.getOutputSockets(); - InputSocket a = inputs.get(0); - InputSocket b = inputs.get(1); - OutputSocket c = outputs.get(0); + InputSocket a = inputs.get(0); + InputSocket b = inputs.get(1); + OutputSocket c = outputs.get(0); int[] sz = {256, 256}; - a.setValue(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(1))); - b.setValue(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(2))); + a.getValue().ifPresent(m -> m.set(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(1)))); + b.getValue().ifPresent(m -> m.set(new Mat(2, sz, opencv_core.CV_8U, Scalar.all(2)))); for (int i = 0; i < 1000; i++) { addition.perform(); } Mat expectedResult = new Mat(2, sz, opencv_core.CV_8U, Scalar.all(3)); - assertTrue(isMatEqual((Mat) c.getValue().get(), expectedResult)); + assertTrue(isMatEqual(c.getValue().get().getCpu(), expectedResult)); } } diff --git a/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java b/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java index b119685574..01313abe33 100644 --- a/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java +++ b/core/src/test/java/edu/wpi/grip/core/serialization/ProjectTest.java @@ -4,6 +4,7 @@ import edu.wpi.grip.core.AdditionOperation; import edu.wpi.grip.core.Connection; import edu.wpi.grip.core.ManualPipelineRunner; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.OperationMetaData; import edu.wpi.grip.core.Pipeline; import edu.wpi.grip.core.PipelineRunner; @@ -274,19 +275,20 @@ public void testPerformSerializedPipelineWithMats() throws Exception { serializeAndDeserialize(); Step step1 = pipeline.getSteps().get(0); - InputSocket a = (InputSocket) step1.getInputSockets().get(0); - InputSocket b = (InputSocket) step1.getInputSockets().get(1); - OutputSocket sum = (OutputSocket) step1.getOutputSockets().get(0); + InputSocket a = (InputSocket) step1.getInputSockets().get(0); + InputSocket b = (InputSocket) step1.getInputSockets().get(1); + OutputSocket sum = (OutputSocket) step1.getOutputSockets().get(0); - a.setValue(new Mat(1, 1, CV_32F, new Scalar(1234.5))); - b.setValue(new Mat(1, 1, CV_32F, new Scalar(6789.0))); + a.getValue().get().set(new Mat(1, 1, CV_32F, new Scalar(1234.5))); + b.getValue().get().set(new Mat(1, 1, CV_32F, new Scalar(6789.0))); + a.flagChanged(); pipelineRunner.runPipeline(); Mat diff = new Mat(); Mat expected = new Mat(1, 1, CV_32F, new Scalar(1234.5 + 6789.0)); - compare(expected, sum.getValue().get(), diff, CMP_NE); + compare(expected, sum.getValue().get().getCpu(), diff, CMP_NE); assertEquals("Deserialized pipeline with Mat operations did not produce the expected sum.", 0, countNonZero(diff)); } @@ -301,8 +303,8 @@ public void testSerializePipelineWithSource() throws Exception { serializeAndDeserialize(); final ImageFileSource sourceDeserialized = (ImageFileSource) pipeline.getSources().get(0); - Files.gompeiJpegFile.assertSameImage((Mat) sourceDeserialized.createOutputSockets().get(0) - .getValue().get()); + Files.gompeiJpegFile.assertSameImage( + (MatWrapper) sourceDeserialized.createOutputSockets().get(0).getValue().get()); } @Test diff --git a/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java b/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java index bd3f0931a6..1d0b0cbe24 100644 --- a/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java +++ b/core/src/test/java/edu/wpi/grip/core/sockets/MockInputSocketFactory.java @@ -1,11 +1,13 @@ package edu.wpi.grip.core.sockets; +import edu.wpi.grip.core.cuda.NullCudaDetector; + import com.google.common.eventbus.EventBus; public class MockInputSocketFactory extends InputSocketImpl.FactoryImpl { public MockInputSocketFactory(EventBus eventBus) { - super(eventBus); + super(eventBus, new NullCudaDetector()); } } diff --git a/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java index 29447211f4..2eaac028cc 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java @@ -281,7 +281,7 @@ static class MockFrameGrabber extends FrameGrabber { for (int y = 0; y < frameIdx.rows(); y++) { for (int x = 0; x < frameIdx.cols(); x++) { for (int z = 0; z < frameIdx.channels(); z++) { - frameIdx.putDouble(new int[]{y, x, z}, y + x + z); + frameIdx.putDouble(new long[]{y, x, z}, y + x + z); } } } diff --git a/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java index 8a81abc172..159c256124 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/HttpSourceTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.Pipeline; import edu.wpi.grip.core.http.ContextStore; import edu.wpi.grip.core.http.GripServer; @@ -17,7 +18,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -62,7 +62,7 @@ public void setUp() throws URISyntaxException { @Test public void testPostImage() throws IOException, InterruptedException { - OutputSocket imageSource = source.getOutputSockets().get(0); + OutputSocket imageSource = source.getOutputSockets().get(0); // We have to manually update the output sockets to get the image source.updateOutputSockets(); diff --git a/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java index 7d87e6b8e7..77d8840607 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/ImageFileSourceTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.MockOutputSocketFactory; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.util.Files; @@ -7,7 +8,6 @@ import com.google.common.eventbus.EventBus; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.Before; import org.junit.Test; @@ -40,7 +40,7 @@ public void testLoadImageToMat() throws IOException { final ImageFileSource fileSource = new ImageFileSource(osf, origin -> null, this .imageFile.file); fileSource.initialize(); - OutputSocket outputSocket = fileSource.getOutputSockets().get(0); + OutputSocket outputSocket = fileSource.getOutputSockets().get(0); // Then assertTrue("The output socket's value was empty.", outputSocket.getValue().isPresent()); @@ -53,7 +53,7 @@ public void testReadInTextFile() throws IOException { final ImageFileSource fileSource = new ImageFileSource(osf, origin -> null, this .textFile); fileSource.initialize(); - OutputSocket outputSocket = fileSource.getOutputSockets().get(0); + OutputSocket outputSocket = fileSource.getOutputSockets().get(0); assertTrue("No matrix should have been returned.", outputSocket.getValue().get().empty()); } @@ -71,7 +71,7 @@ public void testCallingInitializeAfterGetOutputSocketUpdatesOutputSocket() throw final ImageFileSource source = new ImageFileSource(osf, origin -> null, this .imageFile.file); // Calling this before loading the image should throw an exception - final OutputSocket imageSource = source.getOutputSockets().get(0); + final OutputSocket imageSource = source.getOutputSockets().get(0); assertTrue("The value should not be present if the source hasn't been initialized", imageSource.getValue().get().empty()); source.initialize(); diff --git a/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java b/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java index 914e7e1805..8aee0d99f5 100644 --- a/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java +++ b/core/src/test/java/edu/wpi/grip/core/sources/MultiImageFileSourceTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.core.sources; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.MockOutputSocketFactory; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.util.Files; @@ -7,7 +8,6 @@ import com.google.common.eventbus.EventBus; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.Before; import org.junit.Test; @@ -54,7 +54,7 @@ public void createMultiImageFileSourceWithTextFile() throws IOException { @Test public void testNextValue() throws Exception { source.next(); - OutputSocket outputSocket = source.getOutputSockets().get(0); + OutputSocket outputSocket = source.getOutputSockets().get(0); source.updateOutputSockets(); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -63,7 +63,7 @@ public void testNextValue() throws Exception { public void testPreviousValue() throws Exception { source.previous(); - OutputSocket outputSocket = source.getOutputSockets().get(0); + OutputSocket outputSocket = source.getOutputSockets().get(0); source.updateOutputSockets(); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -71,7 +71,7 @@ public void testPreviousValue() throws Exception { @Test public void testConstructedWithIndex() { sourceWithIndexSet.updateOutputSockets(); - OutputSocket outputSocket = sourceWithIndexSet.getOutputSockets().get(0); + OutputSocket outputSocket = sourceWithIndexSet.getOutputSockets().get(0); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } @@ -85,7 +85,7 @@ public void testLoadFromProperties() throws Exception { properties); newSource.initialize(); newSource.updateOutputSockets(); - OutputSocket outputSocket = newSource.getOutputSockets().get(0); + OutputSocket outputSocket = newSource.getOutputSockets().get(0); gompeiJpegFile.assertSameImage(outputSocket.getValue().get()); } } diff --git a/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java b/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java index e5513cabf2..a49affe19f 100644 --- a/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java +++ b/core/src/test/java/edu/wpi/grip/core/util/MetaInfReaderTest.java @@ -34,7 +34,7 @@ public void testRead() throws IOException { String input = "# Leading comment\na\nb\n# Comment in the middle\n\nc\nd\ne # Trailing comment"; List expected = ImmutableList.of("a", "b", "c", "d", "e"); InputStream s = new StringBufferInputStream(input); - List out = MetaInfReader.read(s) + List out = MetaInfReader.readLines(s) .collect(Collectors.toList()); assertEquals("Unexpected line result", expected, out); } diff --git a/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java b/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java index d8b196009c..0e3aed0af4 100644 --- a/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java +++ b/core/src/test/java/edu/wpi/grip/core/util/SafeShutdownTest.java @@ -41,7 +41,7 @@ public void tearDown() { @Test public void testSafeShutdownShutsDownIfHandlerThrowsError() throws Exception { try { - SafeShutdown.exit(0, () -> { + SafeShutdown.exit(SafeShutdown.ExitCode.SAFE_SHUTDOWN, () -> { throw new AssertionError("This should not be the exception that appears"); }); } catch (IllegalStateException e) { diff --git a/core/src/test/java/edu/wpi/grip/util/ImageWithData.java b/core/src/test/java/edu/wpi/grip/util/ImageWithData.java index 261a387d62..40b66ff3ef 100644 --- a/core/src/test/java/edu/wpi/grip/util/ImageWithData.java +++ b/core/src/test/java/edu/wpi/grip/util/ImageWithData.java @@ -1,6 +1,7 @@ package edu.wpi.grip.util; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.util.ImageLoadingUtility; import org.bytedeco.javacpp.opencv_core.Mat; @@ -35,6 +36,10 @@ public Mat createMat() { } } + public void assertSameImage(MatWrapper image) { + assertSameImage(image.getCpu()); + } + public void assertSameImage(final Mat image) { // Check that the image that is read in is 2 dimensional assertEquals("Matrix from loaded image did not have expected number of rows.", this.rows, diff --git a/ui/src/main/java/edu/wpi/grip/ui/DeployController.java b/ui/src/main/java/edu/wpi/grip/ui/DeployController.java index 6c0401a8be..387fd4bed3 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/DeployController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/DeployController.java @@ -36,6 +36,7 @@ import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; + import javafx.application.Platform; import javafx.beans.binding.Bindings; import javafx.beans.property.BooleanProperty; @@ -45,6 +46,7 @@ import javafx.scene.control.ProgressIndicator; import javafx.scene.control.TextField; import javafx.scene.control.ToggleButton; + import javax.inject.Inject; /** diff --git a/ui/src/main/java/edu/wpi/grip/ui/Main.java b/ui/src/main/java/edu/wpi/grip/ui/Main.java index b797176548..2975d965cb 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/Main.java +++ b/ui/src/main/java/edu/wpi/grip/ui/Main.java @@ -1,8 +1,11 @@ package edu.wpi.grip.ui; import edu.wpi.grip.core.GripCoreModule; +import edu.wpi.grip.core.GripCudaModule; import edu.wpi.grip.core.GripFileModule; +import edu.wpi.grip.core.Loggers; import edu.wpi.grip.core.PipelineRunner; +import edu.wpi.grip.core.cuda.CudaVerifier; import edu.wpi.grip.core.events.UnexpectedThrowableEvent; import edu.wpi.grip.core.exception.GripServerException; import edu.wpi.grip.core.http.GripServer; @@ -76,22 +79,37 @@ public static void main(String[] args) { @Override public void init() throws IOException { + Loggers.setupLoggers(); parsedArgs = commandLineHelper.parse(getParameters().getRaw()); + // Verify CUDA before using the core module, since that will cause OpenCV to be loaded, + // which will crash the app if we use CUDA and it's not available + GripCudaModule cudaModule = new GripCudaModule(); + CudaVerifier cudaVerifier = Guice.createInjector(cudaModule).getInstance(CudaVerifier.class); + cudaVerifier.verifyCuda(); + if (parsedArgs.hasOption(UICommandLineHelper.HEADLESS_OPTION)) { // If --headless was specified on the command line, // run in headless mode (only use the core module) logger.info("Launching GRIP in headless mode"); - injector = Guice.createInjector(Modules.override(new GripCoreModule(), new GripFileModule(), - new GripSourcesHardwareModule()).with(new GripNetworkModule())); + injector = Guice.createInjector( + Modules.override( + new GripCoreModule(), + new GripFileModule(), + new GripSourcesHardwareModule() + ).with(new GripNetworkModule(), cudaModule)); injector.injectMembers(this); headless = true; } else { // Otherwise, run with both the core and UI modules, and show the JavaFX stage logger.info("Launching GRIP in UI mode"); - injector = Guice.createInjector(Modules.override(new GripCoreModule(), new GripFileModule(), - new GripSourcesHardwareModule()).with(new GripNetworkModule(), new GripUiModule())); + injector = Guice.createInjector( + Modules.override( + new GripCoreModule(), + new GripFileModule(), + new GripSourcesHardwareModule() + ).with(new GripNetworkModule(), new GripUiModule(), cudaModule)); injector.injectMembers(this); notifyPreloader(new Preloader.ProgressNotification(0.15)); @@ -160,7 +178,9 @@ public void start(Stage stage) throws IOException { + "HTTP sources and operations will not work until GRIP is restarted. " + "Continue without HTTP functionality anyway?" ); - alert.showAndWait().filter(ButtonType.NO::equals).ifPresent(bt -> SafeShutdown.exit(1)); + alert.showAndWait() + .filter(ButtonType.NO::equals) + .ifPresent(bt -> SafeShutdown.exit(SafeShutdown.ExitCode.HTTP_SERVER_COULD_NOT_START)); } } @@ -195,8 +215,8 @@ public final void onUnexpectedThrowableEvent(UnexpectedThrowableEvent event) { try { logger.log(Level.SEVERE, "Failed to show exception alert", e); } finally { - SafeShutdown.exit(1); // Ensure we shut down the application if we get an - // exception + // Ensure we shut down the application if we get an exception + SafeShutdown.exit(SafeShutdown.ExitCode.MISC_ERROR); } } } diff --git a/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java b/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java index ed5a1bedd7..00eb1ea063 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/MainWindowController.java @@ -279,7 +279,7 @@ protected void showProjectAboutDialog() throws IOException { protected boolean quit() { if (showConfirmationDialogAndWait()) { pipelineRunner.stopAsync(); - SafeShutdown.exit(0); + SafeShutdown.exit(SafeShutdown.ExitCode.SAFE_SHUTDOWN); return true; } return false; diff --git a/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java b/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java index 176a63006b..c504e3b539 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java +++ b/ui/src/main/java/edu/wpi/grip/ui/codegeneration/data/TPipeline.java @@ -2,6 +2,7 @@ import edu.wpi.grip.core.Connection; import edu.wpi.grip.core.Step; +import edu.wpi.grip.core.sockets.CudaSocket; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.codegeneration.TemplateMethods; @@ -97,6 +98,10 @@ private void set(List pipeSteps) { for (int i = 0; i < pipeSteps.size(); i++) { TStep tStep = this.steps.get(i); for (InputSocket input : pipeSteps.get(i).getInputSockets()) { + // Skip CudaSockets entirely - generated code is purely CPU + if (input instanceof CudaSocket) { + continue; + } TInput tInput; String type = TemplateMethods.parseSocketType(input); if ("Type".equals(type)) { diff --git a/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java b/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java index 23a3c170dc..d1cd86fbca 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java +++ b/ui/src/main/java/edu/wpi/grip/ui/components/PreviousNextButtons.java @@ -6,6 +6,7 @@ import org.controlsfx.control.SegmentedButton; import java.util.function.Consumer; + import javafx.scene.Node; import javafx.scene.control.ToggleButton; import javafx.scene.control.Tooltip; diff --git a/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java b/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java index f5c5cce7fd..56f3315fd6 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/pipeline/OutputSocketController.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.pipeline; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.events.BenchmarkEvent; import edu.wpi.grip.core.events.SocketChangedEvent; import edu.wpi.grip.core.events.SocketPreviewChangedEvent; @@ -25,7 +26,6 @@ import javafx.scene.layout.StackPane; import static com.google.common.base.Preconditions.checkNotNull; -import static org.bytedeco.javacpp.opencv_core.Mat; /** * A JavaFX control that renders an {@link OutputSocket} that is the output of a step. It shows a @@ -96,13 +96,13 @@ public void onSocketChanged(SocketChangedEvent event) { if (!this.socket.getValue().isPresent()) { // No value handlePreview(false); - } else if (!(this.socket.getValue().get() instanceof Mat)) { + } else if (!(this.socket.getValue().get() instanceof MatWrapper)) { // There is a non-image value, which can always be previewed handlePreview(true); } else { // Only allow the image to be previewed if it's previewable boolean previewable = this.socket.getValue() - .map(Mat.class::cast) + .map(MatWrapper.class::cast) .map(ImageBasedPreviewView::isPreviewable) .get(); handlePreview(previewable); diff --git a/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java b/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java index 09f83e9710..d2bdd99d2e 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/pipeline/PipelineController.java @@ -31,6 +31,7 @@ import java.util.NavigableMap; import java.util.TreeMap; import java.util.stream.Collectors; + import javafx.beans.InvalidationListener; import javafx.beans.property.ReadOnlyObjectProperty; import javafx.collections.ObservableList; @@ -43,6 +44,7 @@ import javafx.scene.input.TransferMode; import javafx.scene.layout.HBox; import javafx.scene.layout.VBox; + import javax.annotation.Nullable; import javax.inject.Inject; diff --git a/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java b/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java index 7329810a45..65cdd9d76e 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java +++ b/ui/src/main/java/edu/wpi/grip/ui/pipeline/input/CheckboxInputSocketController.java @@ -1,6 +1,9 @@ package edu.wpi.grip.ui.pipeline.input; +import edu.wpi.grip.core.cuda.AccelerationMode; +import edu.wpi.grip.core.cuda.CudaDetector; import edu.wpi.grip.core.events.SocketChangedEvent; +import edu.wpi.grip.core.sockets.CudaSocket; import edu.wpi.grip.core.sockets.InputSocket; import edu.wpi.grip.ui.pipeline.SocketHandleView; import edu.wpi.grip.ui.util.GripPlatform; @@ -23,12 +26,17 @@ public class CheckboxInputSocketController extends InputSocketController socket) { + CheckboxInputSocketController(SocketHandleView.Factory socketHandleViewFactory, + GripPlatform platform, + AccelerationMode accelerationMode, + CudaDetector cudaDetector, + @Assisted InputSocket socket) { super(socketHandleViewFactory, socket); this.platform = platform; + isCudaAvailable = accelerationMode.isUsingCuda() && cudaDetector.isCompatibleCudaInstalled(); this.checkBox = new CheckBox(); } @@ -42,6 +50,11 @@ public void initialize() { this.getIdentifier().setContentDisplay(ContentDisplay.RIGHT); this.setContent(checkBox); + // Disable if controlling a CUDA socket and CUDA acceleration is unavailable + if (getSocket() instanceof CudaSocket && !isCudaAvailable) { + this.checkBox.setDisable(true); + } + assignSocketValue(getSocket().getValue()); // Add the listener after so that setting the initial value doesn't trigger it. this.checkBox.selectedProperty().addListener(o -> this.getSocket().setValue(this.checkBox diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java index 75b8ecaca2..00d060858b 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/BlobsSocketPreviewView.java @@ -58,7 +58,7 @@ public BlobsSocketPreviewView(GripPlatform platform, OutputSocket s protected void convertImage() { synchronized (this) { final BlobsReport blobsReport = this.getSocket().getValue().get(); - final Mat input = blobsReport.getInput(); + final Mat input = blobsReport.getInput().getCpu(); if (input.channels() == 3) { input.copyTo(tmp); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java index 50a751baba..0531625ab2 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageBasedPreviewView.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.events.RenderEvent; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.util.ImageConverter; @@ -11,7 +12,6 @@ import static org.bytedeco.javacpp.opencv_core.CV_8S; import static org.bytedeco.javacpp.opencv_core.CV_8U; -import static org.bytedeco.javacpp.opencv_core.Mat; /** * Base class for image previews. @@ -58,7 +58,7 @@ protected final int getImageHeight() { * * @return true if the image can be previewed, false if it can't */ - public static boolean isPreviewable(Mat image) { + public static boolean isPreviewable(MatWrapper image) { return (image.channels() == 1) || (image.channels() == 3) && (image.depth() == CV_8U || image.depth() == CV_8S); } diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java index 647cae478e..fb59a3cae7 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/ImageSocketPreviewView.java @@ -1,23 +1,22 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.ui.util.GripPlatform; import javafx.scene.image.Image; -import static org.bytedeco.javacpp.opencv_core.Mat; - /** * A SocketPreviewView that previews sockets containing OpenCV Mats. */ -public class ImageSocketPreviewView extends ImageBasedPreviewView { +public class ImageSocketPreviewView extends ImageBasedPreviewView { private final GripPlatform platform; /** * @param socket An output socket to preview. */ - ImageSocketPreviewView(GripPlatform platform, OutputSocket socket) { + ImageSocketPreviewView(GripPlatform platform, OutputSocket socket) { super(socket); this.platform = platform; this.setContent(imageView); @@ -28,9 +27,9 @@ protected void convertImage() { synchronized (this) { this.getSocket().getValue() .filter(ImageBasedPreviewView::isPreviewable) - .ifPresent(mat -> { + .ifPresent(m -> { platform.runAsSoonAsPossible(() -> { - Image image = imageConverter.convert(mat, getImageHeight()); + Image image = imageConverter.convert(m.getCpu(), getImageHeight()); imageView.setImage(image); }); }); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java index 8b70e3db3a..603e05417e 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/LinesSocketPreviewView.java @@ -69,7 +69,7 @@ protected void convertImage() { synchronized (this) { final LinesReport linesReport = this.getSocket().getValue().get(); final List lines = linesReport.getLines(); - Mat input = linesReport.getInput(); + Mat input = linesReport.getInput().getCpu(); // If there were lines found, draw them on the image before displaying it if (!linesReport.getLines().isEmpty()) { diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java b/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java index faf0677240..c10f20c496 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/RectangleSocketPreviewView.java @@ -58,7 +58,7 @@ protected void convertImage() { synchronized (this) { final RectsReport report = this.getSocket().getValue().get(); final List rectangles = report.getRectangles(); - Mat input = report.getImage(); + Mat input = report.getImage().getCpu(); if (input.channels() == 3) { input.copyTo(tmp); diff --git a/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java b/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java index 2d7017e98f..38a6b677d7 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java +++ b/ui/src/main/java/edu/wpi/grip/ui/preview/SocketPreviewViewFactory.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.composite.BlobsReport; import edu.wpi.grip.core.operations.composite.ContoursReport; import edu.wpi.grip.core.operations.composite.LinesReport; @@ -11,7 +12,6 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import static org.bytedeco.javacpp.opencv_core.Mat; import static org.bytedeco.javacpp.opencv_core.Point; import static org.bytedeco.javacpp.opencv_core.Size; @@ -39,9 +39,9 @@ public class SocketPreviewViewFactory { @SuppressWarnings("unchecked") public SocketPreviewView create(OutputSocket socket) { final SocketPreviewView previewView; - if (socket.getSocketHint().getType() == Mat.class) { - previewView = (SocketPreviewView) new ImageSocketPreviewView(platform, (OutputSocket) - socket); + if (socket.getSocketHint().getType() == MatWrapper.class) { + previewView = (SocketPreviewView) new ImageSocketPreviewView(platform, + (OutputSocket) socket); } else if (socket.getSocketHint().getType() == Point.class || socket.getSocketHint().getType() == Size.class) { previewView = (SocketPreviewView) new PointSizeSocketPreviewView(platform, socket); diff --git a/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java b/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java index 736a31897e..c5e19d7f36 100644 --- a/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java +++ b/ui/src/main/java/edu/wpi/grip/ui/util/ImageConverter.java @@ -1,5 +1,7 @@ package edu.wpi.grip.ui.util; +import edu.wpi.grip.core.MatWrapper; + import com.google.common.primitives.UnsignedBytes; import org.bytedeco.javacpp.opencv_core.Mat; @@ -25,6 +27,9 @@ public final class ImageConverter { private WritableImage image; private IntBuffer pixels; + public Image convert(MatWrapper wrapper, int desiredHeight) { + return convert(wrapper.getCpu(), desiredHeight); + } /** * Convert a BGR-formatted OpenCV {@link Mat} into a JavaFX {@link Image}. JavaFX understands ARGB diff --git a/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java b/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java index d28209e13c..aa9a918fcc 100644 --- a/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java +++ b/ui/src/test/java/edu/wpi/grip/ui/codegeneration/tools/HelperTools.java @@ -4,7 +4,6 @@ import org.bytedeco.javacpp.indexer.UByteIndexer; import org.opencv.core.Core; -import org.opencv.core.CvType; import org.opencv.core.Mat; import org.opencv.core.MatOfByte; import org.opencv.imgcodecs.Imgcodecs; @@ -23,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.opencv.core.CvType.CV_8UC; public class HelperTools { private static final Logger logger = Logger.getLogger(HelperTools.class.getName()); @@ -68,7 +68,7 @@ public static double matAvgDiff(Mat mat1, Mat mat2) { */ public static Mat bytedecoMatToCVMat(org.bytedeco.javacpp.opencv_core.Mat input) { UByteIndexer idxer = input.createIndexer(); - Mat out = new Mat(idxer.rows(), idxer.cols(), CvType.CV_8UC(idxer.channels())); + Mat out = new Mat((int) idxer.rows(), (int) idxer.cols(), CV_8UC((int) idxer.channels())); //Mat out = new Mat(idxer.rows(),idxer.cols(),input.type()); for (int row = 0; row < idxer.rows(); row++) { for (int col = 0; col < idxer.cols(); col++) { diff --git a/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java b/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java index e866ddf8e8..a1b06f1b36 100644 --- a/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java +++ b/ui/src/test/java/edu/wpi/grip/ui/preview/ImageSocketPreviewViewTest.java @@ -1,5 +1,6 @@ package edu.wpi.grip.ui.preview; +import edu.wpi.grip.core.MatWrapper; import edu.wpi.grip.core.operations.network.MockGripNetworkModule; import edu.wpi.grip.core.sockets.OutputSocket; import edu.wpi.grip.core.sockets.SocketHint; @@ -13,7 +14,6 @@ import com.google.inject.Injector; import com.google.inject.util.Modules; -import org.bytedeco.javacpp.opencv_core.Mat; import org.junit.After; import org.junit.Test; import org.testfx.framework.junit.ApplicationTest; @@ -39,9 +39,9 @@ public void start(Stage stage) { final ImageSocketPreviewView imageSocketPreviewView = new ImageSocketPreviewView(new MockGripPlatform(new EventBus()), injector.getInstance(OutputSocket.Factory.class) - .create(new SocketHint.Builder<>(Mat.class) + .create(new SocketHint.Builder<>(MatWrapper.class) .identifier(identifier) - .initialValueSupplier(Files.gompeiJpegFile::createMat) + .initialValueSupplier(() -> MatWrapper.wrap(Files.gompeiJpegFile.createMat())) .build())); final Scene scene = new Scene(imageSocketPreviewView); stage.setScene(scene); diff --git a/ui/ui.gradle.kts b/ui/ui.gradle.kts index b982dec998..95016c7d23 100644 --- a/ui/ui.gradle.kts +++ b/ui/ui.gradle.kts @@ -17,6 +17,12 @@ if (!(project.hasProperty("generation") || project.hasProperty("genonly"))) { } } +val withCuda = project.hasProperty("cuda") || project.hasProperty("WITH_CUDA") + +if (withCuda) { + version = "$version-cuda" +} + dependencies { compile(project(":core")) compile(project(":ui:preloader"))