From ac6817c7e4aefd1aea578a13a5130b50b0989194 Mon Sep 17 00:00:00 2001 From: Mark Lee Date: Wed, 3 Feb 2016 19:53:25 +0000 Subject: [PATCH] Use native direct ByteBuffer instead of Java heap buffer. This removes one full-frame memory copy. --- README.md | 18 +++++ pom.xml | 2 +- .../component/DirectMediaPlayerComponent.java | 9 +-- .../vlcj/player/direct/ByteBufferFactory.java | 54 ++++++++++++++ .../direct/DefaultDirectMediaPlayer.java | 21 ++++-- .../vlcj/player/direct/DirectMediaPlayer.java | 5 +- .../vlcj/player/direct/RenderCallback.java | 5 +- .../player/direct/RenderCallbackAdapter.java | 74 ------------------- .../DirectMediaPlayerComponentTest.java | 26 +++---- .../vlcj/test/direct/DirectTestPlayer.java | 12 ++- 10 files changed, 116 insertions(+), 110 deletions(-) create mode 100644 src/main/java/uk/co/caprica/vlcj/player/direct/ByteBufferFactory.java delete mode 100644 src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallbackAdapter.java diff --git a/README.md b/README.md index e0993d82c..e711d23b4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,23 @@ ![vlcj](https://github.com/caprica/vlcj/raw/master/etc/vlcj-logo.png "vlcj") +vlcj experimental branch +======================== + +This is an experimental branch of vlcj, used to test improvements to the direct +rendering media players. + +Currently this is limited to using direct ByteBuffers to provide access to the +video data. LibVLC can render directly into the ByteBuffer, thereby removing at +least one full frame copy. An application can render the contents of the +ByteBuffer in whatever way it sees fit - e.g. it could copy the data to a +BufferedImage or a PixelWriter. + +There is still one 'extra' copy here, ideally LibVLC would render directly into +the BufferedImage or PixelWriter. + +Nevertheless, initial performance testing shows considerable improvement over +that in vlcj 3.x. + vlcj ==== diff --git a/pom.xml b/pom.xml index 8f4756f54..0d5fe3d80 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ uk.co.caprica vlcj - 3.11.0-SNAPSHOT + 4.0.0-SNAPSHOT vlcj Java Framework for the vlc Media Player. diff --git a/src/main/java/uk/co/caprica/vlcj/component/DirectMediaPlayerComponent.java b/src/main/java/uk/co/caprica/vlcj/component/DirectMediaPlayerComponent.java index 6b54aae8e..47fec4bd1 100644 --- a/src/main/java/uk/co/caprica/vlcj/component/DirectMediaPlayerComponent.java +++ b/src/main/java/uk/co/caprica/vlcj/component/DirectMediaPlayerComponent.java @@ -19,6 +19,7 @@ package uk.co.caprica.vlcj.component; +import java.nio.ByteBuffer; import java.util.Arrays; import org.slf4j.Logger; @@ -32,7 +33,6 @@ import uk.co.caprica.vlcj.player.direct.BufferFormatCallback; import uk.co.caprica.vlcj.player.direct.DirectMediaPlayer; import uk.co.caprica.vlcj.player.direct.RenderCallback; -import uk.co.caprica.vlcj.player.direct.RenderCallbackAdapter; import com.sun.jna.Memory; @@ -214,10 +214,9 @@ protected String[] onGetMediaPlayerFactoryExtraArgs() { * Template method to obtain a render callback implementation. *

* The default behaviour is simply to return this component instance itself so that sub-classes - * may override {@link #display(Memory)}. + * may override {@link #display(DirectMediaPlayer, ByteBuffer[], BufferFormat)}. *

- * A sub-class may provide any implementation of {@link RenderCallback} - including - * {@link RenderCallbackAdapter}. + * A sub-class may provide any implementation of {@link RenderCallback}. * * @return render callback implementation */ @@ -402,7 +401,7 @@ public void endOfSubItems(MediaPlayer mediaPlayer) { // === RenderCallback ======================================================= @Override - public void display(DirectMediaPlayer mediaPlayer, Memory[] nativeBuffers, BufferFormat bufferFormat) { + public void display(DirectMediaPlayer mediaPlayer, ByteBuffer[] nativeBuffers, BufferFormat bufferFormat) { // Default implementation does nothing, sub-classes should override this or // provide their own implementation of a RenderCallback } diff --git a/src/main/java/uk/co/caprica/vlcj/player/direct/ByteBufferFactory.java b/src/main/java/uk/co/caprica/vlcj/player/direct/ByteBufferFactory.java new file mode 100644 index 000000000..b748355fc --- /dev/null +++ b/src/main/java/uk/co/caprica/vlcj/player/direct/ByteBufferFactory.java @@ -0,0 +1,54 @@ +package uk.co.caprica.vlcj.player.direct; + +import sun.nio.ch.DirectBuffer; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Factory for creating property aligned native byte buffers. + */ +public class ByteBufferFactory { + + /** + * Alignment suitable for use by LibVLC video callbacks. + */ + private static final int LIBVLC_ALIGNMENT = 32; + + /** + * Allocate a properly aligned native byte buffer, suitable for use by the LibVLC video + * callbacks. + * + * @param capacity size of the buffer + * @return aligned byte buffer + */ + public static ByteBuffer allocateAlignedBuffer(int capacity) { + return allocateAlignedBuffer(capacity, LIBVLC_ALIGNMENT); + } + + /** + * Allocate a property aligned native byte buffer. + *

+ * Original credit: http://psy-lob-saw.blogspot.co.uk/2013/01/direct-memory-alignment-in-java.html + * + * @param capacity size of the buffer + * @param alignment alignment + * @return aligned byte buffer + */ + public static ByteBuffer allocateAlignedBuffer(int capacity, int alignment) { + ByteBuffer result; + ByteBuffer buffer = ByteBuffer.allocateDirect(capacity + alignment); + long address = ((DirectBuffer) buffer).address(); + if ((address & (alignment - 1)) == 0) { + buffer.limit(capacity); + result = buffer.slice().order(ByteOrder.nativeOrder()); + } + else { + int newPosition = (int) (alignment - (address & (alignment - 1))); + buffer.position(newPosition); + buffer.limit(newPosition + capacity); + result = buffer.slice().order(ByteOrder.nativeOrder()); + } + return result; + } +} diff --git a/src/main/java/uk/co/caprica/vlcj/player/direct/DefaultDirectMediaPlayer.java b/src/main/java/uk/co/caprica/vlcj/player/direct/DefaultDirectMediaPlayer.java index 2b953f421..79f83fc4f 100644 --- a/src/main/java/uk/co/caprica/vlcj/player/direct/DefaultDirectMediaPlayer.java +++ b/src/main/java/uk/co/caprica/vlcj/player/direct/DefaultDirectMediaPlayer.java @@ -19,11 +19,14 @@ package uk.co.caprica.vlcj.player.direct; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.concurrent.Semaphore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.nio.ch.DirectBuffer; import uk.co.caprica.vlcj.binding.LibVlc; import uk.co.caprica.vlcj.binding.internal.libvlc_display_callback_t; import uk.co.caprica.vlcj.binding.internal.libvlc_instance_t; @@ -127,7 +130,12 @@ public class DefaultDirectMediaPlayer extends DefaultMediaPlayer implements Dire /** * Native memory buffers, one for each plane. */ - private Memory[] nativeBuffers; + private ByteBuffer[] nativeBuffers; + + /** + * Native memory pointers to each byte buffer. + */ + private Pointer[] pointers; /** * Create a new media player. @@ -179,7 +187,7 @@ public final BufferFormat getBufferFormat() { } @Override - public final Memory[] lock() { + public final ByteBuffer[] lock() { semaphore.acquireUninterruptibly(); return nativeBuffers; } @@ -218,9 +226,12 @@ public int format(PointerByReference opaque, PointerByReference chroma, IntByRef // Memory must be aligned correctly (on a 32-byte boundary) for the libvlc // API functions (extra bytes are allocated to allow for enough memory if // the alignment needs to be changed) - nativeBuffers = new Memory[bufferFormat.getPlaneCount()]; + nativeBuffers = new ByteBuffer[bufferFormat.getPlaneCount()]; + pointers = new Pointer[bufferFormat.getPlaneCount()]; for(int i = 0; i < bufferFormat.getPlaneCount(); i ++ ) { - nativeBuffers[i] = new Memory(pitchValues[i] * lineValues[i] + 32).align(32); + ByteBuffer buffer = ByteBufferFactory.allocateAlignedBuffer(pitchValues[i] * lineValues[i]); + nativeBuffers[i] = buffer; + pointers[i] = Pointer.createConstant(((DirectBuffer) buffer).address()); } logger.trace("format finished"); return pitchValues.length; @@ -260,7 +271,7 @@ public Pointer lock(Pointer opaque, PointerByReference planes) { semaphore.acquireUninterruptibly(); logger.trace("acquired"); // Set the pre-allocated buffers to use for each plane - planes.getPointer().write(0, nativeBuffers, 0, nativeBuffers.length); + planes.getPointer().write(0, pointers, 0, pointers.length); logger.trace("lock finished"); return null; } diff --git a/src/main/java/uk/co/caprica/vlcj/player/direct/DirectMediaPlayer.java b/src/main/java/uk/co/caprica/vlcj/player/direct/DirectMediaPlayer.java index 45b71aaeb..cca12fb63 100644 --- a/src/main/java/uk/co/caprica/vlcj/player/direct/DirectMediaPlayer.java +++ b/src/main/java/uk/co/caprica/vlcj/player/direct/DirectMediaPlayer.java @@ -19,10 +19,13 @@ package uk.co.caprica.vlcj.player.direct; +import com.sun.jna.Pointer; import uk.co.caprica.vlcj.player.MediaPlayer; import com.sun.jna.Memory; +import java.nio.ByteBuffer; + /** * Specification for a media player that provides direct access to the video frame data. *

@@ -52,7 +55,7 @@ public interface DirectMediaPlayer extends MediaPlayer { * * @return native memory buffers, may be null */ - Memory[] lock(); + ByteBuffer[] lock(); /** * Unlock the native memory buffers. diff --git a/src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallback.java b/src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallback.java index 9e9dde756..1db56a6b4 100644 --- a/src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallback.java +++ b/src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallback.java @@ -20,6 +20,9 @@ package uk.co.caprica.vlcj.player.direct; import com.sun.jna.Memory; +import com.sun.jna.Pointer; + +import java.nio.ByteBuffer; /** * Specification for a component that wishes to be called back to process video frames. @@ -40,5 +43,5 @@ public interface RenderCallback { * @param nativeBuffers video data for one frame * @param bufferFormat information about the format of the buffer used */ - void display(DirectMediaPlayer mediaPlayer, Memory[] nativeBuffers, BufferFormat bufferFormat); + void display(DirectMediaPlayer mediaPlayer, ByteBuffer[] nativeBuffers, BufferFormat bufferFormat); } diff --git a/src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallbackAdapter.java b/src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallbackAdapter.java deleted file mode 100644 index 66675c185..000000000 --- a/src/main/java/uk/co/caprica/vlcj/player/direct/RenderCallbackAdapter.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * This file is part of VLCJ. - * - * VLCJ is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * VLCJ is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with VLCJ. If not, see . - * - * Copyright 2009-2016 Caprica Software Limited. - */ - -package uk.co.caprica.vlcj.player.direct; - -import com.sun.jna.Memory; - -/** - * A render call-back adapter implementation that fills an array of RGB integer data for an entire - * video frame. - *

- * The media player must be sending pixels in the RV32 format. - *

- * If you simply want access to the native memory buffer you should consider sub-classing - * {@link RenderCallback} directly rather than using this class. - *

- * This is probably the most inefficient implementation possible of a render callback, - * ordinarily the video data should be written directly to some other construct (like a texture). - */ -public abstract class RenderCallbackAdapter implements RenderCallback { - - /** - * Video data buffer. - */ - private final int[] rgbBuffer; - - /** - * Create a new render call-back. - * - * @param rgbBuffer video data buffer - */ - public RenderCallbackAdapter(int[] rgbBuffer) { - this.rgbBuffer = rgbBuffer; - } - - @Override - public void display(DirectMediaPlayer mediaPlayer, Memory[] nativeBuffer, BufferFormat bufferFormat) { - nativeBuffer[0].getByteBuffer(0L, nativeBuffer[0].size()).asIntBuffer().get(rgbBuffer(), 0, bufferFormat.getHeight() * bufferFormat.getWidth()); - onDisplay(mediaPlayer, rgbBuffer()); - } - - /** - * Get the video data buffer. - * - * @return video buffer - */ - public int[] rgbBuffer() { - return rgbBuffer; - } - - /** - * Template method invoked when a new frame of video data is ready. - * - * @param mediaPlayer media player - * @param rgbBuffer video data buffer - */ - protected abstract void onDisplay(DirectMediaPlayer mediaPlayer, int[] rgbBuffer); -} diff --git a/src/test/java/uk/co/caprica/vlcj/test/component/DirectMediaPlayerComponentTest.java b/src/test/java/uk/co/caprica/vlcj/test/component/DirectMediaPlayerComponentTest.java index f3251e878..93ed4bb82 100644 --- a/src/test/java/uk/co/caprica/vlcj/test/component/DirectMediaPlayerComponentTest.java +++ b/src/test/java/uk/co/caprica/vlcj/test/component/DirectMediaPlayerComponentTest.java @@ -27,6 +27,7 @@ import java.awt.GraphicsEnvironment; import java.awt.RenderingHints; import java.awt.image.BufferedImage; +import java.nio.ByteBuffer; import javax.swing.JComponent; import javax.swing.JFrame; @@ -38,7 +39,6 @@ import uk.co.caprica.vlcj.player.direct.BufferFormatCallback; import uk.co.caprica.vlcj.player.direct.DirectMediaPlayer; import uk.co.caprica.vlcj.player.direct.RenderCallback; -import uk.co.caprica.vlcj.player.direct.RenderCallbackAdapter; import uk.co.caprica.vlcj.player.direct.format.RV32BufferFormat; import uk.co.caprica.vlcj.test.VlcjTest; @@ -144,7 +144,15 @@ public BufferFormat getBufferFormat(int sourceWidth, int sourceHeight) { mediaPlayerComponent = new DirectMediaPlayerComponent(bufferFormatCallback) { @Override protected RenderCallback onGetRenderCallback() { - return new TestRenderCallbackAdapter(); + return new RenderCallback() { + @Override + public void display(DirectMediaPlayer mediaPlayer, ByteBuffer[] nativeBuffers, BufferFormat bufferFormat) { + // FIXME test! + int[] rgbBuffer = nativeBuffers[0].asIntBuffer().array(); + image.setRGB(0, 0, width, height, rgbBuffer, 0, width); + panel.repaint(); + } + }; } }; @@ -165,18 +173,4 @@ private void start(String mrl) { // One line of vlcj code to play the media... mediaPlayerComponent.getMediaPlayer().playMedia(mrl); } - - private class TestRenderCallbackAdapter extends RenderCallbackAdapter { - - private TestRenderCallbackAdapter() { - super(new int[width * height]); - } - - @Override - protected void onDisplay(DirectMediaPlayer mediaPlayer, int[] rgbBuffer) { - // Simply copy buffer to the image and repaint - image.setRGB(0, 0, width, height, rgbBuffer, 0, width); - panel.repaint(); - } - } } diff --git a/src/test/java/uk/co/caprica/vlcj/test/direct/DirectTestPlayer.java b/src/test/java/uk/co/caprica/vlcj/test/direct/DirectTestPlayer.java index 83fc3b988..2c7a9ecb8 100644 --- a/src/test/java/uk/co/caprica/vlcj/test/direct/DirectTestPlayer.java +++ b/src/test/java/uk/co/caprica/vlcj/test/direct/DirectTestPlayer.java @@ -33,6 +33,7 @@ import java.awt.image.BufferedImage; import java.awt.image.DataBufferInt; import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; import java.util.Arrays; import javax.swing.ImageIcon; @@ -44,7 +45,7 @@ import uk.co.caprica.vlcj.player.direct.BufferFormat; import uk.co.caprica.vlcj.player.direct.BufferFormatCallback; import uk.co.caprica.vlcj.player.direct.DirectMediaPlayer; -import uk.co.caprica.vlcj.player.direct.RenderCallbackAdapter; +import uk.co.caprica.vlcj.player.direct.RenderCallback; import uk.co.caprica.vlcj.player.direct.format.RV32BufferFormat; import uk.co.caprica.vlcj.test.VlcjTest; @@ -163,14 +164,11 @@ public void paint(Graphics g) { } } - private final class TestRenderCallback extends RenderCallbackAdapter { - - public TestRenderCallback() { - super(((DataBufferInt) image.getRaster().getDataBuffer()).getData()); - } + private final class TestRenderCallback implements RenderCallback { @Override - public void onDisplay(DirectMediaPlayer mediaPlayer, int[] data) { + public void display(DirectMediaPlayer mediaPlayer, ByteBuffer[] nativeBuffers, BufferFormat bufferFormat) { + // FIXME make this work again // The image data could be manipulated here... /* RGB to GRAYScale conversion example */