From 79a4b73b8917d8c2cd3609751257ef80ed52eac4 Mon Sep 17 00:00:00 2001 From: Stephen von Takach Date: Thu, 24 Aug 2023 12:13:16 +1000 Subject: [PATCH] add device details and update readme --- README.md | 57 ++++++++++++++++++++++++++++++++------ spec/v4l2_spec.cr | 20 +++++++++---- src/v4l2.cr | 5 ++++ src/v4l2/device_details.cr | 37 +++++++++++++++++++++++++ src/v4l2/pixel_format.cr | 3 +- src/v4l2/video.cr | 20 +++++++++++-- 6 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 src/v4l2/device_details.cr diff --git a/README.md b/README.md index bc1fb00..1afe10a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ Basic v4l2 bindings for extracting frames as close to realtime from a video device on linux - ## Installation 1. Add the dependency to your `shard.yml`: @@ -17,7 +16,53 @@ Basic v4l2 bindings for extracting frames as close to realtime from a video devi ## Usage -So multiple processes can access the video stream, it's useful to create a loopback device. +Capture frames with the lowest possible latency + +```crystal +# low latency device access +require "v4l2" +# use libav for converting the frames to a useable format +require "ffmpeg" + +# grab a handle to the device +video = Video.new(Path["/dev/video0"]) + +# you can then grab the names of the devices +details = video.device_details +details.name # => "USB2.0 HD UVC WebCam: USB2.0 HD" + +# You can grab the supported formats, resolutions and frame rates +formats = vid.supported_formats +formats.each(&.frame_sizes.each(&.frame_rate)) + +# Once you've selected the appropriate configuration you can set it +# this is the optimal selection for me (YUYV 4:2:2 - 640x480 @ 30fps) +format = pixels[1].frame_sizes[1].frame_rate +video.set_format(format) + +# then you need to configure how many buffers you need +# once all your buffers are full it starts dropping frames +# set to 1 if you only want the most up to date frame +video.request_buffers(1) + +# configure the desired output format +rgb_frame = FFmpeg::Frame.new(format.width, format.height, :rgb48Le) +convert = FFmpeg::SWScale.new(format.width, format.height, :yuyv422, output_format: :rgb48Le) + +# now you can start processing frames +video.stream do |buffer| + # buffer is returned to the driver when this block returns + # so either perform processing here or copy the Bytes + + # process the video + v4l2_frame = FFmpeg::Frame.new(format.width, format.height, :yuyv422, buffer: buffer) + convert.scale(v4l2_frame, rgb_frame) + + # channel.send rgb_frame +end +``` + +So multiple processes can access the video stream, it's useful to create a loopback device (if you have issues with this) * 1 process for processing images * 1 process for viewing the stream etc @@ -35,13 +80,7 @@ v4l2-ctl --list-devices # stream the video to the loopback (for low latency multiple app access) ffmpeg -f v4l2 -i /dev/video0 -f v4l2 /dev/video4 # OR -gst-launch-1.0 v4l2src device=/dev/video0 ! v4l2sink device=/dev/video2 -``` - -See the specs for basic usage - -```crystal -require "v4l2" +gst-launch-1.0 v4l2src device=/dev/video0 ! v4l2sink device=/dev/video4 ``` ## Development diff --git a/spec/v4l2_spec.cr b/spec/v4l2_spec.cr index 22b316b..11b7b04 100644 --- a/spec/v4l2_spec.cr +++ b/spec/v4l2_spec.cr @@ -18,6 +18,12 @@ module V4L2 vid.buffer.index.should eq 0_u32 end + it "gets the device name" do + vid = Video.new(Path["/dev/video0"]) + details = vid.device_details + (details.name.size > 0).should be_true + end + it "can stream and scale" do vid = Video.new(Path["/dev/video0"]) pixels = vid.supported_formats @@ -25,12 +31,13 @@ module V4L2 # highest resolution + framerate combo format = pixels[1].frame_sizes[1].frame_rate - vid.set_format(format).request_buffers(2).allocate_mmap_buffers(2).start_stream + vid.set_format(format).request_buffers(1) # setup the ffmpeg image format conversion - v4l2_frame = FFmpeg::Frame.new(format.width, format.height, :yuyv422) + # v4l2_frame = FFmpeg::Frame.new(format.width, format.height, :yuyv422) rgb_frame = FFmpeg::Frame.new(format.width, format.height, :rgb48Le) - convert = FFmpeg::SWScale.new(v4l2_frame, rgb_frame) + # convert = FFmpeg::SWScale.new(v4l2_frame, rgb_frame) + convert = FFmpeg::SWScale.new(format.width, format.height, :yuyv422, output_format: :rgb48Le) spawn do sleep 3 @@ -38,13 +45,14 @@ module V4L2 end count = 0 - vid.raw_stream do |buffer| + vid.stream do |buffer| print '+' count += 1 # process the frame using ffmpeg / libav - # TODO:: benchmark copy vs creating new frame using this buffer - buffer.copy_to v4l2_frame.buffer + # creating new frame using this buffer is faster than copying + # buffer.copy_to v4l2_frame.buffer + v4l2_frame = FFmpeg::Frame.new(format.width, format.height, :yuyv422, buffer: buffer) convert.scale(v4l2_frame, rgb_frame) end diff --git a/src/v4l2.cr b/src/v4l2.cr index 5afc4c1..947236c 100644 --- a/src/v4l2.cr +++ b/src/v4l2.cr @@ -28,6 +28,11 @@ module V4L2 ioc(LibC::IOC_WRITE, type.ord.to_u32, nr.to_u32, size.to_u32) end + def self.ior(type : Char, nr : Int, size : Int) + ioc(LibC::IOC_READ, type.ord.to_u32, nr.to_u32, size.to_u32) + end + + VIDIOC_QUERYCAP = V4L2.ior('V', 0, sizeof(LibV4l2::Capability)) VIDIOC_S_FMT = V4L2.iorw('V', 5, sizeof(LibV4l2::Format)) VIDIOC_ENUM_FMT = V4L2.iorw('V', 2, sizeof(LibV4l2::Fmtdesc)) VIDIOC_ENUM_FRAMESIZES = V4L2.iorw('V', 74, sizeof(LibV4l2::Frmsizeenum)) diff --git a/src/v4l2/device_details.cr b/src/v4l2/device_details.cr new file mode 100644 index 0000000..a3194cb --- /dev/null +++ b/src/v4l2/device_details.cr @@ -0,0 +1,37 @@ +class V4L2::DeviceDetails + def initialize(@capability : LibV4l2::Capability) + capability = @capability + + str = capability.driver.to_slice + str_end = str.index(&.zero?) + @driver = String.new(str[0...str_end]) + + str = capability.card.to_slice + str_end = str.index(&.zero?) + @card = String.new(str[0...str_end]) + + str = capability.bus_info.to_slice + str_end = str.index(&.zero?) + @bus_info = String.new(str[0...str_end]) + end + + getter driver : String + getter card : String + getter bus_info : String + + def name + card + end + + def version + @capability.version + end + + def capabilities + @capability.capabilities + end + + def device_caps + @capability.device_caps + end +end diff --git a/src/v4l2/pixel_format.cr b/src/v4l2/pixel_format.cr index de31382..048a99a 100644 --- a/src/v4l2/pixel_format.cr +++ b/src/v4l2/pixel_format.cr @@ -1,7 +1,8 @@ require "./frame_size" class V4L2::PixelFormat - def initialize(@io : IO::FileDescriptor, format : LibV4l2::Fmtdesc) + def initialize(@io : IO::FileDescriptor, @format : LibV4l2::Fmtdesc) + format = @format @id = format.pixelformat @code = PixelFormat.pixel_format_chars(format.pixelformat) @index = format.index diff --git a/src/v4l2/video.cr b/src/v4l2/video.cr index d0f69c8..a6ebaf3 100644 --- a/src/v4l2/video.cr +++ b/src/v4l2/video.cr @@ -37,6 +37,13 @@ class V4L2::Video formats end + def device_details + capability = LibV4l2::Capability.new + ret = LibC.ioctl(@io.fd, VIDIOC_QUERYCAP.to_u64, pointerof(capability)) + raise RuntimeError.from_errno "failed to obtain device details (#{ret})" if ret < 0 + DeviceDetails.new(capability) + end + def set_format(format_id : UInt32, width : UInt32, height : UInt32, buffer_type : BufferType = BufferType::VIDEO_CAPTURE) fmt = LibV4l2::Format.new fmt.type = buffer_type.value @@ -75,6 +82,12 @@ class V4L2::Video if ret < 0 raise "failed to request buffers #{count}, #{buffer_type}, #{memory_type} (#{ret})" end + + case memory_type + when .mmap? + allocate_mmap_buffers(count, buffer_type) + end + self end @@ -82,8 +95,6 @@ class V4L2::Video count : Int, buffer_type : BufferType = BufferType::VIDEO_CAPTURE ) - raise ArgumentError.new("requires at least 2 buffers, requested #{count}") if count < 2 - length = 0_u32 buffers = (0...count).map do |index| buffer = reset_buffer @@ -110,6 +121,7 @@ class V4L2::Video end def start_stream(buffer_type : BufferType = BufferType::VIDEO_CAPTURE) + return self if @streaming type = buffer_type.value ret = LibC.ioctl(@io.fd, VIDIOC_STREAMON.to_u64, pointerof(type)) raise "stream failed to start (#{ret})" if ret < 0 @@ -117,10 +129,12 @@ class V4L2::Video self end - def raw_stream(timeout : Time::Span = 2.seconds, buffer_type : BufferType = BufferType::VIDEO_CAPTURE, & : Bytes ->) + def stream(timeout : Time::Span = 2.seconds, buffer_type : BufferType = BufferType::VIDEO_CAPTURE, & : Bytes ->) buffers = @buffers raise "must allocate buffers" unless buffers + start_stream(buffer_type) unless @streaming + while @streaming && !@io.closed? @io.wait_readable timeout