diff --git a/gradle.properties b/gradle.properties index 40278fbfe..0544b58d4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,7 @@ version_caffeine=3.1.8 version_clikt=4.2.2 version_commonsmath3=3.6.1 version_cottontaildb=0.16.7 +version_jaffree=2024.08.29 version_javacv=1.5.10 version_javalin=6.3.0 version_jdbc_postgres=42.7.4 diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/Resolver.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/Resolver.kt index c0bb50044..3150f4939 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/Resolver.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/Resolver.kt @@ -13,8 +13,9 @@ interface Resolver { * Attempts to resolve the provided [RetrievableId] to a [Resolvable] using this [Resolver]. * * @param id The [RetrievableId] to resolve. + * @param suffix The suffix of the filename. * @return [Resolvable] or null, if [RetrievableId] could not be resolved. */ - fun resolve(id: RetrievableId) : Resolvable? + fun resolve(id: RetrievableId, suffix: String) : Resolvable? } \ No newline at end of file diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/impl/DiskResolver.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/impl/DiskResolver.kt index 4343c9e7d..f5a6a527b 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/impl/DiskResolver.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/resolver/impl/DiskResolver.kt @@ -28,14 +28,13 @@ class DiskResolver : ResolverFactory { */ override fun newResolver(schema: Schema, parameters: Map): Resolver { val location = Paths.get(parameters["location"] ?: "./thumbnails/${schema.name}") - val mimeType = MimeType.valueOf(parameters["mimeType"] ?: "JPG") - return Instance(location, mimeType) + return Instance(location) } /** * The [Resolver] generated by this [DiskResolver]. */ - private class Instance(private val location: Path, private val mimeType: MimeType) : Resolver { + private class Instance(private val location: Path) : Resolver { init { /* Make sure, directory exists. */ if (!Files.exists(this.location)) { @@ -47,20 +46,19 @@ class DiskResolver : ResolverFactory { * Resolves the provided [RetrievableId] to a [Resolvable] using this [Resolver]. * * @param id The [RetrievableId] to resolve. + * @param suffix The suffix of the filename. * @return [Resolvable] or null, if [RetrievableId] could not be resolved. */ - override fun resolve(id: RetrievableId): Resolvable = DiskResolvable(id) - + override fun resolve(id: RetrievableId, suffix: String): Resolvable = DiskResolvable(id, suffix) /** * A [Resolvable] generated by this [DiskResolver]. */ - inner class DiskResolvable(override val retrievableId: RetrievableId) : Resolvable { - val path: Path - get() = this@Instance.location.resolve("$retrievableId.${this@Instance.mimeType.fileExtension}") - override val mimeType: MimeType - get() = this@Instance.mimeType - + inner class DiskResolvable(override val retrievableId: RetrievableId, suffix: String) : Resolvable { + val path: Path = this@Instance.location.resolve("${retrievableId}.$suffix") + override val mimeType: MimeType by lazy { + MimeType.getMimeType(this.path) ?: MimeType.UNKNOWN + } override fun exists(): Boolean = Files.exists(this.path) override fun openInputStream(): InputStream = Files.newInputStream(this.path, StandardOpenOption.READ) override fun openOutputStream(): OutputStream = Files.newOutputStream(this.path, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE) diff --git a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/source/file/MimeType.kt b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/source/file/MimeType.kt index 5669a31bc..ece74d09f 100644 --- a/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/source/file/MimeType.kt +++ b/vitrivr-engine-core/src/main/kotlin/org/vitrivr/engine/core/source/file/MimeType.kt @@ -59,12 +59,8 @@ enum class MimeType(val fileExtension: String, val mimeType: String, val mediaTy OFF("off", "application/3d-off", MediaType.MESH), GLTF("gltf", "model/gltf+json", MediaType.MESH), - - - //Unknown type - UNKNOWN("", "", MediaType.NONE) - ; + UNKNOWN("", "", MediaType.NONE); companion object { fun getMimeType(fileName: String): MimeType? = try { @@ -80,6 +76,6 @@ enum class MimeType(val fileExtension: String, val mimeType: String, val mediaTy null } - val allValid = MimeType.values().filter { it != UNKNOWN }.toSet() + val allValid = entries.filter { it != UNKNOWN }.toSet() } } \ No newline at end of file diff --git a/vitrivr-engine-index/build.gradle b/vitrivr-engine-index/build.gradle index fa4523819..d15b08a00 100644 --- a/vitrivr-engine-index/build.gradle +++ b/vitrivr-engine-index/build.gradle @@ -28,6 +28,9 @@ dependencies { implementation group: 'org.bytedeco', name: 'javacv', version: version_javacv implementation group: 'org.bytedeco', name: 'ffmpeg', version: version_ffmpeg, classifier: project.ext.javacppPlatform + /** Jaffree for external ffmpeg*/ + implementation group: 'com.github.kokorin.jaffree', name: 'jaffree', version: version_jaffree + /** ScrImage (used for image resizing). */ implementation group: 'com.sksamuel.scrimage', name: 'scrimage-core', version: version_scrimage diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/decode/FFmpegVideoDecoder.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/decode/FFmpegVideoDecoder.kt new file mode 100644 index 000000000..4d7cb698e --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/decode/FFmpegVideoDecoder.kt @@ -0,0 +1,303 @@ +package org.vitrivr.engine.index.decode + +import com.github.kokorin.jaffree.StreamType +import com.github.kokorin.jaffree.ffmpeg.* +import com.github.kokorin.jaffree.ffprobe.FFprobe +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.channels.Channel.Factory.RENDEZVOUS +import kotlinx.coroutines.channels.ProducerScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.runBlocking +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.model.content.element.AudioContent +import org.vitrivr.engine.core.model.content.element.ImageContent +import org.vitrivr.engine.core.model.relationship.Relationship +import org.vitrivr.engine.core.model.retrievable.Ingested +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.attributes.ContentAuthorAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.SourceAttribute +import org.vitrivr.engine.core.model.retrievable.attributes.time.TimeRangeAttribute +import org.vitrivr.engine.core.operators.ingest.Decoder +import org.vitrivr.engine.core.operators.ingest.DecoderFactory +import org.vitrivr.engine.core.operators.ingest.Enumerator +import org.vitrivr.engine.core.source.MediaType +import org.vitrivr.engine.core.source.Metadata +import org.vitrivr.engine.core.source.Source +import org.vitrivr.engine.core.source.file.FileSource +import java.awt.image.BufferedImage +import java.nio.ShortBuffer +import java.nio.file.Path +import java.util.* +import java.util.concurrent.TimeUnit + +/** + * A [Decoder] that can decode [ImageContent] and [AudioContent] from a [Source] of [MediaType.VIDEO]. + * + * Based on Jaffree FFmpeg wrapper, which spawns a new FFmpeg process for each [Source]. + * + * @author Luca Rossetto + * @author Ralph Gasser + * @version 1.0.0 + */ +class FFmpegVideoDecoder : DecoderFactory { + + override fun newDecoder(name: String, input: Enumerator, context: IndexContext): Decoder { + val video = context[name, "video"]?.let { it.lowercase() == "true" } != false + val audio = context[name, "audio"]?.let { it.lowercase() == "true" } != false + val timeWindowMs = context[name, "timeWindowMs"]?.toLongOrNull() ?: 500L + val ffmpegPath = context[name, "ffmpegPath"]?.let { Path.of(it) } + + return Instance(input, context, video, audio, timeWindowMs, ffmpegPath, name) + } + + private class Instance( + override val input: Enumerator, + private val context: IndexContext, + private val video: Boolean = true, + private val audio: Boolean = true, + private val timeWindowMs: Long = 500L, + private val ffmpegPath: Path? = null, + private val name: String + ) : Decoder { + + /** [KLogger] instance. */ + private val logger: KLogger = KotlinLogging.logger {} + + private val ffprobe: FFprobe + get() = if (this.ffmpegPath != null) FFprobe.atPath(this.ffmpegPath) else FFprobe.atPath() + + private val ffmpeg: FFmpeg + get() = if (this.ffmpegPath != null) FFmpeg.atPath(this.ffmpegPath) else FFmpeg.atPath() + + override fun toFlow(scope: CoroutineScope): Flow = channelFlow { + this@Instance.input.toFlow(scope).collect { sourceRetrievable -> + /* Extract source. */ + val source = sourceRetrievable.filteredAttribute(SourceAttribute::class.java)?.source ?: return@collect + if (source.type != MediaType.VIDEO) { + logger.debug { "In flow: Skipping source ${source.name} (${source.sourceId}) because it is not of type VIDEO." } + return@collect + } + + val probeResult = ffprobe.setShowStreams(true).also { + if (source is FileSource) { + it.setInput(source.path) + } else { + it.setInput(source.newInputStream()) + } + }.execute() + + /* Extract metadata. */ + val videoStreamInfo = probeResult.streams.find { it.codecType == StreamType.VIDEO } + if (videoStreamInfo != null) { + source.metadata[Metadata.METADATA_KEY_VIDEO_FPS] = videoStreamInfo.avgFrameRate.toDouble() + source.metadata[Metadata.METADATA_KEY_AV_DURATION] = (videoStreamInfo.duration * 1000f).toLong() + source.metadata[Metadata.METADATA_KEY_IMAGE_WIDTH] = videoStreamInfo.width + source.metadata[Metadata.METADATA_KEY_IMAGE_HEIGHT] = videoStreamInfo.height + } + + val audioStreamInfo = probeResult.streams.find { it.codecType == StreamType.AUDIO } + if (audioStreamInfo != null) { + source.metadata[Metadata.METADATA_KEY_AUDIO_CHANNELS] = audioStreamInfo.channels + source.metadata[Metadata.METADATA_KEY_AUDIO_SAMPLERATE] = audioStreamInfo.sampleRate + source.metadata[Metadata.METADATA_KEY_AUDIO_SAMPLESIZE] = audioStreamInfo.sampleFmt + } + + /* Create consumer. */ + val consumer = InFlowFrameConsumer(this, sourceRetrievable) + + /* Execute. */ + try { + var output = FrameOutput.withConsumerAlpha(consumer).disableStream(StreamType.SUBTITLE).disableStream(StreamType.DATA) + if (!this@Instance.video) { + output = output.disableStream(StreamType.VIDEO) + } + if (!this@Instance.audio) { + output = output.disableStream(StreamType.AUDIO) + } + if (source is FileSource) { + this@Instance.ffmpeg.addInput(UrlInput.fromPath(source.path)).addOutput(output).execute() + } else { + source.newInputStream().use { + this@Instance.ffmpeg.addInput(PipeInput.pumpFrom(it)).addOutput(output).execute() + } + } + + + /* Emit final frames. */ + if (!consumer.isEmpty()) { + consumer.emit() + } + + /* Emit source retrievable. */ + send(sourceRetrievable) + } catch (e: Throwable) { + logger.error(e) { "Error while decoding source ${source.name} (${source.sourceId})." } + } + } + }.buffer(capacity = RENDEZVOUS, onBufferOverflow = BufferOverflow.SUSPEND) + + + /** + * A [FrameConsumer] that emits [Retrievable]s to the downstream [channel]. + */ + private inner class InFlowFrameConsumer(private val channel: ProducerScope, val source: Retrievable) : FrameConsumer { + + /** The video [Stream] processed by this [InFlowFrameConsumer]. */ + var videoStream: Stream? = null + private set + + /** The audio [Stream] processed by this [InFlowFrameConsumer]. */ + var audioStream: Stream? = null + private set + + /** The end of the time window. */ + var windowEnd = TimeUnit.MILLISECONDS.toMicros(this@Instance.timeWindowMs) + private set + + /** Flag indicating, that video is ready to be emitted. */ + var videoReady = false + + /** Flag indicating, that audio is ready to be emitted. */ + var audioReady = false + + /** [List] of grabbed [BufferedImage]s. */ + val imageBuffer: List> = LinkedList() + + /** [List] of grabbed [ShortBuffer]s. */ + val audioBuffer: List> = LinkedList() + + /** + * Returns true if both the image and audio buffer are empty. + */ + fun isEmpty(): Boolean = this.imageBuffer.isEmpty() && this.audioBuffer.isEmpty() + + /** + * Initializes this [InFlowFrameConsumer]. + * + * @param streams List of [Stream]s to initialize the [InFlowFrameConsumer] with. + */ + override fun consumeStreams(streams: MutableList) { + this.videoStream = streams.firstOrNull { it.type == Stream.Type.VIDEO } + this.audioStream = streams.firstOrNull { it.type == Stream.Type.AUDIO } + } + + /** + * Consumes a single [Frame]. + * + * @param frame [Frame] to consume. + */ + override fun consume(frame: Frame?) = runBlocking { + if (frame == null) return@runBlocking + val stream = when (frame.streamId) { + this@InFlowFrameConsumer.audioStream?.id -> this@InFlowFrameConsumer.audioStream!! + this@InFlowFrameConsumer.videoStream?.id -> this@InFlowFrameConsumer.videoStream!! + else -> return@runBlocking + } + val timestamp = ((1_000_000 * frame.pts) / stream.timebase) + when (stream.type) { + Stream.Type.VIDEO -> { + (this@InFlowFrameConsumer.imageBuffer as LinkedList).add(frame.image!! to timestamp) + if (timestamp >= this@InFlowFrameConsumer.windowEnd) { + this@InFlowFrameConsumer.videoReady = true + } + } + Stream.Type.AUDIO -> { + val samples = ShortBuffer.wrap(frame.samples.map { (it shr 16).toShort() }.toShortArray()) + (this@InFlowFrameConsumer.audioBuffer as LinkedList).add(samples to timestamp) + if (timestamp >= this@InFlowFrameConsumer.windowEnd) { + this@InFlowFrameConsumer.audioReady = true + } + } + else -> {} + } + + /* If enough frames have been collected, emit them. */ + if (this@InFlowFrameConsumer.videoReady && this@InFlowFrameConsumer.audioReady) { + emit() + + /* Reset counters and flags. */ + this@InFlowFrameConsumer.videoReady = !(this@InFlowFrameConsumer.videoStream != null && this@Instance.video) + this@InFlowFrameConsumer.audioReady = !(this@InFlowFrameConsumer.audioStream != null && this@Instance.audio) + + /* Update window end. */ + this@InFlowFrameConsumer.windowEnd += TimeUnit.MILLISECONDS.toMicros(this@Instance.timeWindowMs) + } + } + + /** + * Emits a single [Retrievable] to the downstream [channel]. + */ + suspend fun emit() { + /* Audio samples. */ + var audioSize = 0 + val emitImage = mutableListOf() + val emitAudio = mutableListOf() + + /* Drain buffers. */ + (this.imageBuffer as LinkedList).removeIf { + if (it.second <= this.windowEnd) { + emitImage.add(it.first) + true + } else { + false + } + } + (this.audioBuffer as LinkedList).removeIf { + if (it.second <= this.windowEnd) { + audioSize += it.first.limit() + emitAudio.add(it.first) + true + } else { + false + } + } + + /* Prepare ingested with relationship to source. */ + val ingested = Ingested(UUID.randomUUID(), "SEGMENT", false) + this.source.filteredAttribute(SourceAttribute::class.java)?.let { ingested.addAttribute(it) } + ingested.addRelationship(Relationship.ByRef(ingested, "partOf", source, false)) + ingested.addAttribute( + TimeRangeAttribute( + this.windowEnd - TimeUnit.MILLISECONDS.toMicros(this@Instance.timeWindowMs), + this.windowEnd, + TimeUnit.MICROSECONDS + ) + ) + + /* Prepare and append audio content element. */ + if (emitAudio.isNotEmpty()) { + val samples = ShortBuffer.allocate(audioSize) + for (frame in emitAudio) { + frame.clear() + samples.put(frame) + } + samples.clear() + val audio = this@Instance.context.contentFactory.newAudioContent( + this.audioStream!!.channels.toShort(), + this.audioStream!!.sampleRate.toInt(), + samples + ) + ingested.addContent(audio) + ingested.addAttribute(ContentAuthorAttribute(audio.id, name)) + } + + /* Prepare and append image content element. */ + for (image in emitImage) { + val imageContent = this@Instance.context.contentFactory.newImageContent(image) + ingested.addContent(imageContent) + ingested.addAttribute(ContentAuthorAttribute(imageContent.id, name)) + } + + logger.debug { "Emitting ingested ${ingested.id} with ${emitImage.size} images and ${emitAudio.size} audio samples: ${ingested.id}" } + + /* Emit ingested. */ + this.channel.send(ingested) + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/ThumbnailExporter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/ThumbnailExporter.kt index 03d959bd9..591522e02 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/ThumbnailExporter.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/ThumbnailExporter.kt @@ -58,38 +58,47 @@ class ThumbnailExporter : ExporterFactory { require(mimeType in SUPPORTED) { "ThumbnailExporter only support image formats JPEG and PNG." } } + /** [KLogger] instance. */ + private val logger: KLogger = KotlinLogging.logger {} override fun toFlow(scope: CoroutineScope): Flow = this.input.toFlow(scope).onEach { retrievable -> - val resolvable = this.context.resolver.resolve(retrievable.id) + try { - val contentIds = this.contentSources?.let { - retrievable.filteredAttribute(ContentAuthorAttribute::class.java)?.getContentIds(it) - } + val resolvable = this.context.resolver.resolve(retrievable.id, ".${this.mimeType.fileExtension}") - val content = retrievable.content.filterIsInstance().filter { contentIds?.contains(it.id) ?: true } - if (resolvable != null && content.isNotEmpty()) { - val writer = when (mimeType) { - MimeType.JPEG, - MimeType.JPG -> JpegWriter() - MimeType.PNG -> PngWriter() - else -> throw IllegalArgumentException("Unsupported mime type $mimeType") + val contentIds = this.contentSources?.let { + retrievable.filteredAttribute(ContentAuthorAttribute::class.java)?.getContentIds(it) } - logger.debug { "Generating thumbnail(s) for ${retrievable.id} with ${retrievable.type} and resolution $maxResolution. Storing it with ${resolvable::class.simpleName}." } + val content = + retrievable.content.filterIsInstance().filter { contentIds?.contains(it.id) ?: true } + if (resolvable != null && content.isNotEmpty()) { + val writer = when (mimeType) { + MimeType.JPEG, + MimeType.JPG -> JpegWriter() - content.forEach { cnt -> - val imgBytes = ImmutableImage.fromAwt(cnt.content).let { - if (it.width > it.height) { - it.scaleToWidth(maxResolution) - } else { - it.scaleToHeight(maxResolution) - } - }.bytes(writer) + MimeType.PNG -> PngWriter() + else -> throw IllegalArgumentException("Unsupported mime type $mimeType") + } - resolvable.openOutputStream().use { - it.write(imgBytes) + logger.debug { "Generating thumbnail(s) for ${retrievable.id} with ${retrievable.type} and resolution $maxResolution. Storing it with ${resolvable::class.simpleName}." } + + content.forEach { cnt -> + val imgBytes = ImmutableImage.fromAwt(cnt.content).let { + if (it.width > it.height) { + it.scaleToWidth(maxResolution) + } else { + it.scaleToHeight(maxResolution) + } + }.bytes(writer) + + resolvable.openOutputStream().use { + it.write(imgBytes) + } } } + } catch (e: Exception) { + logger.error(e){"Error during thumbnail creation"} } } } diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/VideoPreviewExporter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/VideoPreviewExporter.kt index ee80caec2..faa993fcc 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/VideoPreviewExporter.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/VideoPreviewExporter.kt @@ -11,7 +11,6 @@ import kotlinx.coroutines.flow.onEach import org.bytedeco.javacpp.PointerScope import org.bytedeco.javacv.FFmpegFrameGrabber import org.bytedeco.javacv.Java2DFrameConverter -import org.bytedeco.javacv.Java2DFrameUtils import org.vitrivr.engine.core.context.IndexContext import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.attributes.SourceAttribute @@ -78,7 +77,7 @@ class VideoPreviewExporter : ExporterFactory { override fun toFlow(scope: CoroutineScope): Flow = this.input.toFlow(scope).onEach { retrievable -> val source = retrievable.filteredAttribute(SourceAttribute::class.java)?.source ?: return@onEach if (source.type == MediaType.VIDEO) { - val resolvable = this.context.resolver.resolve(retrievable.id) + val resolvable = this.context.resolver.resolve(retrievable.id, ".${this.mimeType.fileExtension}") if (resolvable != null) { val writer = when (mimeType) { MimeType.JPEG, diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/WaveExporter.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/WaveExporter.kt new file mode 100644 index 000000000..f64d44043 --- /dev/null +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/exporters/WaveExporter.kt @@ -0,0 +1,60 @@ +package org.vitrivr.engine.index.exporters + +import io.github.oshai.kotlinlogging.KLogger +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.onEach +import org.vitrivr.engine.core.context.IndexContext +import org.vitrivr.engine.core.model.content.element.AudioContent +import org.vitrivr.engine.core.model.retrievable.Retrievable +import org.vitrivr.engine.core.model.retrievable.attributes.ContentAuthorAttribute +import org.vitrivr.engine.core.operators.Operator +import org.vitrivr.engine.core.operators.general.Exporter +import org.vitrivr.engine.core.operators.general.ExporterFactory +import org.vitrivr.engine.index.util.WaveUtilities +import java.io.IOException + +private val logger: KLogger = KotlinLogging.logger {} + +/** + * An [Exporter] that generates wave files from audio samples. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +class WaveExporter: ExporterFactory { + /** + * Creates a new [Exporter] instance from this [ThumbnailExporter]. + * + * @param name The name of the [Exporter] + * @param input The [Operator] to acting as an input. + * @param context The [IndexContext] to use. + */ + override fun newExporter(name: String, input: Operator, context: IndexContext): Exporter { + return Instance(input, context, context[name, "contentSources"]?.split(",")?.toSet() ) + } + + /** + * The [Exporter] generated by this [WaveExporter]. + */ + private class Instance(override val input: Operator, private val context: IndexContext, private val contentSources:Set?) : Exporter { + + override fun toFlow(scope: CoroutineScope): Flow = this.input.toFlow(scope).onEach { retrievable -> + try { + val resolvable = this.context.resolver.resolve(retrievable.id, ".wav") + val contentIds = this.contentSources?.let { + retrievable.filteredAttribute(ContentAuthorAttribute::class.java)?.getContentIds(it) + } + val content = retrievable.content.filterIsInstance().filter { contentIds?.contains(it.id) ?: true } + if (resolvable != null && content.isNotEmpty()) { + resolvable.openOutputStream().use { + WaveUtilities.export(content, it) + } + } + } catch (e: IOException) { + logger.error(e){ "IO exception during wave creation." } + } + } + } +} \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/util/WaveUtilities.kt b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/util/WaveUtilities.kt index bd69083c8..31878d53e 100644 --- a/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/util/WaveUtilities.kt +++ b/vitrivr-engine-index/src/main/kotlin/org/vitrivr/engine/index/util/WaveUtilities.kt @@ -1,9 +1,10 @@ package org.vitrivr.engine.index.util import org.vitrivr.engine.core.model.content.element.AudioContent +import java.io.ByteArrayOutputStream +import java.io.OutputStream import java.nio.ByteBuffer import java.nio.ByteOrder -import java.nio.channels.ByteChannel import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption @@ -12,7 +13,7 @@ import java.nio.file.StandardOpenOption * A collection of utilities for handling WAVE files. * * @author Ralph Gasser - * @version 1.0.0 + * @version 1.1.0 */ object WaveUtilities { /** @@ -22,22 +23,36 @@ object WaveUtilities { * @param path The path to the file. * */ - fun export(content: List, path: Path) { + fun export(content: List, path: Path) = Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING).use { + export(content, it) + } + + /** + * Exports a list of [AudioContent] as WAVE file (.wav). + * + * @param content List of [AudioContent] to export. + * @param stream The [OutputStream] to write to. + * + */ + fun export(content: List, stream: OutputStream) { if (content.isEmpty()) return - Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW).use { channel -> - /* Write audio data. */ - var bytes = 0 - channel.position(44) - content.forEach { audio -> - val buffer = ByteBuffer.allocate(audio.size).order(ByteOrder.LITTLE_ENDIAN) - buffer.asShortBuffer().put(audio.content) - bytes += channel.write(buffer) - } - /* Write header. */ - channel.position(0) - writeWaveHeader(channel, content.first().channels, content.first().samplingRate, bytes) + /* Write samples. */ + val samples = ByteArrayOutputStream() + var bytes = 0 + content.forEach { audio -> + val buffer = ByteBuffer.allocate(audio.size).order(ByteOrder.LITTLE_ENDIAN) + buffer.asShortBuffer().put(audio.content) + samples.write(buffer.array()) + bytes += buffer.array().size } + + /* Write header. */ + val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN) + writeWaveHeader(header, content.first().channels, content.first().samplingRate, bytes) + + stream.write(header.array()) + samples.writeTo(stream) } /** @@ -48,17 +63,25 @@ object WaveUtilities { */ fun export(content: AudioContent, path: Path) = export(listOf(content), path) + /** + * Exports a single [AudioContent] as WAVE file (.wav). + * + * @param content [AudioContent] to export. + * @param stream The [OutputStream] to export to + */ + fun export(content: AudioContent, stream: OutputStream) = export(listOf(content), stream) + /** * Writes the WAV header to the ByteBuffer. * + * @param buffer The ByteBuffer to write the header to. * @param channels The number of channels in the WAV file. * @param sampleRate Sample rate of the output file. * @param length Length in bytes of the frames data */ - private fun writeWaveHeader(channel: ByteChannel, channels: Short, sampleRate: Int, length: Int) { + private fun writeWaveHeader(buffer: ByteBuffer, channels: Short, sampleRate: Int, length: Int) { /* Length of the subChunk2. */ val subChunk2Length: Int = length * channels * Short.SIZE_BYTES /* Number of bytes for audio data: NumSamples * NumChannels * BytesPerSample. */ - val buffer = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN) /* RIFF Chunk. */ buffer.put("RIFF".toByteArray(Charsets.US_ASCII)) @@ -78,6 +101,5 @@ object WaveUtilities { /* Data chunk */ buffer.put("data".toByteArray(Charsets.US_ASCII)) /* Begin of the data chunk. */ buffer.putInt(subChunk2Length) /* Length of the data chunk. */ - channel.write(buffer.flip()) } } \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.general.ExporterFactory b/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.general.ExporterFactory index 0420c873e..857aad4b1 100644 --- a/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.general.ExporterFactory +++ b/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.general.ExporterFactory @@ -1,2 +1,3 @@ org.vitrivr.engine.index.exporters.ThumbnailExporter +org.vitrivr.engine.index.exporters.WaveExporter org.vitrivr.engine.index.exporters.VideoPreviewExporter \ No newline at end of file diff --git a/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.ingest.DecoderFactory b/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.ingest.DecoderFactory index 78e195114..6f25574ae 100644 --- a/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.ingest.DecoderFactory +++ b/vitrivr-engine-index/src/main/resources/META-INF/services/org.vitrivr.engine.core.operators.ingest.DecoderFactory @@ -1,2 +1,3 @@ org.vitrivr.engine.index.decode.VideoDecoder +org.vitrivr.engine.index.decode.FFmpegVideoDecoder org.vitrivr.engine.index.decode.ImageDecoder \ No newline at end of file diff --git a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelPreviewExporter.kt b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelPreviewExporter.kt index 237ebca52..a3cbd207f 100644 --- a/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelPreviewExporter.kt +++ b/vitrivr-engine-module-m3d/src/main/kotlin/org/vitrivr/engine/model3d/ModelPreviewExporter.kt @@ -12,17 +12,17 @@ import org.vitrivr.engine.core.model.mesh.texturemodel.Model3d import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.EntopyCalculationMethod import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.EntropyOptimizerStrategy import org.vitrivr.engine.core.model.mesh.texturemodel.util.entropyoptimizer.OptimizerOptions +import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f import org.vitrivr.engine.core.model.retrievable.Retrievable import org.vitrivr.engine.core.model.retrievable.attributes.SourceAttribute -import org.vitrivr.engine.core.model.mesh.texturemodel.util.types.Vec3f import org.vitrivr.engine.core.operators.Operator import org.vitrivr.engine.core.operators.general.Exporter import org.vitrivr.engine.core.operators.general.ExporterFactory import org.vitrivr.engine.core.source.MediaType import org.vitrivr.engine.core.source.file.MimeType import org.vitrivr.engine.model3d.lwjglrender.render.RenderOptions -import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions import org.vitrivr.engine.model3d.lwjglrender.util.texturemodel.entroopyoptimizer.ModelEntropyOptimizer +import org.vitrivr.engine.model3d.lwjglrender.window.WindowOptions import org.vitrivr.engine.model3d.renderer.ExternalRenderer import java.awt.image.BufferedImage import java.io.ByteArrayOutputStream @@ -45,10 +45,10 @@ private val logger: KLogger = KotlinLogging.logger {} */ class ModelPreviewExporter : ExporterFactory { companion object { - val SUPPORTED = setOf(MimeType.GLTF) + val SUPPORTED_INPUT = setOf(MimeType.GLTF) /** Set of supported output formats. */ - val OUTPUT_FORMAT = setOf("gif", "jpg") + val SUPPORTED_OUTPUT = setOf(MimeType.GIF, MimeType.JPG, MimeType.JPEG) /** * Renders a preview of the given model as a JPEG image. @@ -186,7 +186,7 @@ class ModelPreviewExporter : ExporterFactory { } } ?: MimeType.GLTF val distance = context[name, "distance"]?.toFloatOrNull() ?: 1f - val format = context[name, "format"] ?: "gif" + val format = MimeType.valueOf(context[name, "format"]?.uppercase() ?: "GIF") val views = context[name, "views"]?.toIntOrNull() ?: 30 logger.debug { @@ -202,12 +202,12 @@ class ModelPreviewExporter : ExporterFactory { private val maxResolution: Int, mimeType: MimeType, private val distance: Float, - private val format: String, + private val format: MimeType, private val views: Int ) : Exporter { init { - require(mimeType in SUPPORTED) { "ModelPreviewExporter only supports models of format GLTF." } - require(format in OUTPUT_FORMAT) { "ModelPreviewExporter only supports exporting a gif of jpg." } + require(mimeType in SUPPORTED_INPUT) { "ModelPreviewExporter only supports models of format GLTF." } + require(this.format in SUPPORTED_OUTPUT) { "ModelPreviewExporter only supports exporting a gif of jpg." } } override fun toFlow(scope: CoroutineScope): Flow { @@ -216,7 +216,7 @@ class ModelPreviewExporter : ExporterFactory { val source = retrievable.filteredAttribute(SourceAttribute::class.java)?.source ?: return@onEach if (source.type == MediaType.MESH) { - val resolvable = this.context.resolver.resolve(retrievable.id) + val resolvable = this.context.resolver.resolve(retrievable.id, ".${this.format.fileExtension}") val model = retrievable.content[0].content as Model3d if (resolvable != null) { @@ -225,15 +225,20 @@ class ModelPreviewExporter : ExporterFactory { } source.newInputStream().use { input -> - if (format == "jpg") { - val preview: BufferedImage = renderPreviewJPEG(model, renderer, this.distance) - resolvable.openOutputStream().use { output -> - ImageIO.write(preview, "jpg", output) + when (format) { + MimeType.JPG, + MimeType.JPEG -> { + val preview: BufferedImage = renderPreviewJPEG(model, renderer, this.distance) + resolvable.openOutputStream().use { output -> + ImageIO.write(preview, "jpg", output) + } + } + MimeType.GIF -> { + val frames = createFramesForGif(model, renderer, this.views, this.distance) + val gif = createGif(frames, 50) + resolvable.openOutputStream().use { output -> output.write(gif!!.toByteArray()) } } - } else { // format == "gif" - val frames = createFramesForGif(model, renderer, this.views, this.distance) - val gif = createGif(frames, 50) - resolvable.openOutputStream().use { output -> output.write(gif!!.toByteArray()) } + else -> throw IllegalArgumentException("Unsupported mime type $format") } } } diff --git a/vitrivr-engine-server/build.gradle b/vitrivr-engine-server/build.gradle index 654a8ca13..637882c49 100644 --- a/vitrivr-engine-server/build.gradle +++ b/vitrivr-engine-server/build.gradle @@ -11,6 +11,7 @@ dependencies { api project(':vitrivr-engine-module-features') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ api project(':vitrivr-engine-module-cottontaildb') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ api project(':vitrivr-engine-module-pgvector') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ + api project(':vitrivr-engine-module-jsonl') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ api project(':vitrivr-engine-module-fes') /* TODO: This dependency is not necessary and only here to facilitate easy testing. */ /** Clikt & JLine */ diff --git a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/api/rest/handlers/FetchExportData.kt b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/api/rest/handlers/FetchExportData.kt index 1f070dfbb..c44de7ab9 100644 --- a/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/api/rest/handlers/FetchExportData.kt +++ b/vitrivr-engine-server/src/main/kotlin/org/vitrivr/engine/server/api/rest/handlers/FetchExportData.kt @@ -37,7 +37,7 @@ fun fetchExportData(ctx: Context, schema: Schema) { } /* Try to resolve resolvable for retrievable ID. */ - val resolvable = schema.getExporter(exporterName)?.resolver?.resolve(retrievableId) + val resolvable = schema.getExporter(exporterName)?.resolver?.resolve(retrievableId, ".jpg") if (resolvable == null) { ctx.status(404) ctx.json(ErrorStatus("Failed to resolve data."))