Handle media files that has an audio stream whose duration is shorter than format=duration

subed/subed-waveform.el

@@ -251,6 +251,97 @@ WIDTH and HEIGHT are given in pixels."
 
 (defvar-local subed-waveform-file-duration-ms-cache nil "If non-nil, duration of current file in milliseconds.")
 
+(defun subed-waveform-convert-ffprobe-tags-duration-to-ms (duration)
+  "Return milliseconds as an integer for DURATION.
+
+DURATION must be a string of the format HH:MM:SS.MMMM.
+
+Example:
+
+00:00:03.003000000 -> 3003
+00:00:03.00370000 -> 3004"
+  (unless (string-match "\\([0-9]\\{2\\}\\):\\([0-9]\\{2\\}\\):\\([0-9]\\{2\\}\\)\\.\\([0-9]+\\)" duration)
+    (error "The duration is not well formatted."))
+  (let ((hour (match-string 1 duration))
+        (minute (match-string 2 duration))
+        (seconds (match-string 3 duration))
+        (milliseconds (match-string 4 duration)))
+    (+
+     (* (string-to-number hour) 3600000)
+     (* (string-to-number minute) 60000)
+     (* (string-to-number seconds) 1000)
+     (* (string-to-number (concat "0." milliseconds)) 1000))))
+
+(defun subed-waveform-ffprobe-duration-ms (filename)
+  "Use ffprobe to get duration of audio stream in milliseconds of FILENAME."
+  (let ((json
+         (json-read-from-string
+          (with-temp-buffer
+            (call-process
+             subed-waveform-ffprobe-executable nil t nil
+             "-v" "error"
+             "-print_format" "json"
+             "-show_streams"
+             "-show_format"
+             filename)
+            (buffer-string)))))
+    ;; Check that the file has at least one audio stream.
+    (when (eq (seq-find
+               (lambda (stream)
+                 (equal (alist-get 'codec_type stream) "audio"))
+               (alist-get 'streams json))
+              0)
+      (error "The provided file doesn't have an audio stream."))
+    (cond
+     ;; If the file has one stream and it is an audio stream, we can
+     ;; get the duration from format=duration
+     ;;
+     ;; nb_streams equals the number of streams in the media file.
+     ((and (eq (alist-get 'nb_streams (alist-get 'format json)) 1)
+           (equal (alist-get
+                   'codec_type
+                   (seq-first (alist-get 'streams json)))
+                  "audio"))
+      (* 1000 (string-to-number
+               (alist-get 'duration (alist-get 'format json)))))
+     ;; If the file has more than one stream and only one audio
+     ;; stream, return the duration of the audio stream.
+     ((and (> (alist-get 'nb_streams (alist-get 'format json)) 1)
+           (eq (length (seq-filter
+                        (lambda (stream)
+                          (equal (alist-get 'codec_type stream) "audio"))
+                        (alist-get 'streams json)))
+               1))
+      (cond
+       ((or
+         (string-match "\\.mkv\\'" filename)
+         (string-match "\\.webm\\'" filename))
+        (subed-waveform-convert-ffprobe-tags-duration-to-ms
+         (alist-get
+          'DURATION
+          (alist-get
+           'tags
+           (seq-find
+            (lambda (stream)
+              (equal (alist-get 'codec_type stream) "audio"))
+            (alist-get 'streams json))))))
+       (t
+        (* 1000
+           (string-to-number
+            (alist-get
+             'duration
+             (seq-find
+              (lambda (stream)
+                (equal (alist-get 'codec_type stream) "audio"))
+              (alist-get 'streams json))))))))
+     ;; TODO: Some media files might have multiple audio streams
+     ;; (e.g. multiple languages). When the media file has multiple
+     ;; audio streams, prompt the user for the audio stream. The audio
+     ;; stream selected by the user must be stored in a buffer-local
+     ;; variable so that ffmpeg knows the audio stream from which the
+     ;; waveforms are created.
+     )))
+
 (defun subed-waveform-file-duration-ms (&optional filename)
   "Return the duration of FILENAME in milliseconds."
   (cond
@@ -259,14 +350,8 @@ WIDTH and HEIGHT are given in pixels."
       subed-waveform-file-duration-ms-cache))
    (subed-waveform-ffprobe-executable
     (setq subed-waveform-file-duration-ms-cache
-          (* 1000
-             (string-to-number
-              (with-temp-buffer
-                (call-process
-                 subed-waveform-ffprobe-executable nil t nil "-v" "error" "-show_entries" "format=duration"
-                 "-of" "default=noprint_wrappers=1:nokey=1"
-                 (or filename (subed-media-file)))
-                (buffer-string)))))
+          (subed-waveform-ffprobe-duration-ms
+           (or filename (subed-media-file))))
     (if (> subed-waveform-file-duration-ms-cache 0)
         subed-waveform-file-duration-ms-cache
       ;; mark as invalid

tests/test-subed-mpv.el

@@ -1,4 +1,4 @@
-;; -*- eval: (buttercup-minor-mode) -*-
+;; -*- lexical-binding: t; eval: (buttercup-minor-mode) -*-
 
 (load-file "./tests/undercover-init.el")
 (require 'subed-mpv)

tests/test-subed-srt.el

@@ -1,4 +1,4 @@
-;; -*- eval: (buttercup-minor-mode) -*-
+;; -*- lexical-binding: t; eval: (buttercup-minor-mode) -*-
 
 (load-file "./tests/undercover-init.el")
 (require 'subed-srt)

tests/test-subed-tsv.el

@@ -1,4 +1,4 @@
-;; -*- eval: (buttercup-minor-mode) -*-
+;; -*- lexical-binding: t; eval: (buttercup-minor-mode) -*-
 
 (add-to-list 'load-path "./subed")
 (require 'subed)

tests/test-subed-waveform.el

@@ -0,0 +1,217 @@
+;; -*- eval: (buttercup-minor-mode); lexical-binding: t -*-
+
+(require 'subed-waveform)
+
+(cl-defun create-sample-media-file-1-audio-stream (&key
+                                                   path
+                                                   duration-audio-stream)
+  "Create a sample media file with one audio stream
+
+PATH is the absolute path for the output file. It must be a
+string.
+
+DURATION is the number of seconds for the media file. It must be
+a string."
+  (call-process
+   ;; The ffmpeg command shown below can create files with the
+   ;; extensions shown below (tested using ffmpeg version
+   ;; 4.4.2-0ubuntu0.22.04.1)
+   ;; + audio extensions: wav ogg mp3 opus m4a
+   ;; + video extensions: mkv mp4 webm avi ts ogv"
+   "ffmpeg"
+   nil
+   nil
+   nil
+   "-v" "error"
+   "-y"
+   ;; We use lavfi to create the audio stream
+   "-f" "lavfi" "-i" (concat "sine=frequency=1000:duration=" duration-audio-stream)
+   path))
+
+(cl-defun create-sample-media-file-1-audio-stream-1-video-stream (&key
+                                                                  path
+                                                                  duration-video-stream
+                                                                  duration-audio-stream)
+  "Create a sample media file with 1 audio stream and 1 video stream.
+
+PATH is the absolute path for the output file. It must be a
+string.
+
+AUDIO-DURATION is the duration in seconds for the audio
+stream. It must be a string.
+
+VIDEO-DURATION is the duration in seconds for the video stream. It
+must be a string."
+  (call-process
+   ;; The ffmpeg command shown below can create files with the
+   ;; extensions shown below (tested using ffmpeg version
+   ;; 4.4.2-0ubuntu0.22.04.1)
+   ;; + audio extensions: wav ogg mp3 opus m4a
+   ;; + video extensions: mkv mp4 webm avi ts ogv"
+   "ffmpeg"
+   nil
+   nil
+   nil
+   "-v" "error"
+   "-y"
+   ;; Create the video stream
+   "-f" "lavfi" "-i" (concat "testsrc=size=100x100:duration=" duration-video-stream)
+   ;; Create the audio stream
+   "-f" "lavfi" "-i" (concat "sine=frequency=1000:duration=" duration-audio-stream)
+   path))
+
+(describe "waveform"
+  (describe "Get duration in milliseconds of a file with a single audio stream"
+    (let (;; `duration-audio-stream' is the duration in seconds for
+          ;; the media file that is used inside the tests.  When
+          ;; `duration-audio-stream' is an integer, ffprobe might
+          ;; report a duration that is slightly greater, so we can't
+          ;; expect the duration reported by ffprobe to be equal to
+          ;; the duration that we passed to ffmpeg when creating the
+          ;; sample media file. For this reason, we define the
+          ;; variables `duration-lower-boundary' and
+          ;; `duration-upper-boundary' to set a tolerance to the
+          ;; reported value by ffprobe.
+          ;;
+          ;; When `duration-audio-stream' changes, the variables
+          ;; `duration-lower-boundary' and
+          ;; `duration-upper-boundary' should be set accordingly."
+          (duration-audio-stream "3")
+          (duration-lower-boundary 3000)
+          (duration-upper-boundary 4000))
+      (describe "audio file"
+        (it "extension .wav"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.wav"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.wav")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .ogg"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.ogg"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.ogg")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .mp3"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.mp3"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.mp3")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .opus"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.opus"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.opus")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .m4a"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.m4a"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.m4a")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary))))
+      (describe "video file"
+        (it "extension .mkv"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.mkv"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.mkv")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .mp4"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.mp4"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.mp4")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .webm"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.webm"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.webm")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .avi"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.avi"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.avi")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .ts"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.ts"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.ts")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary)))
+        (it "extension .ogv"
+          (create-sample-media-file-1-audio-stream
+           :path "/tmp/a.ogv"
+           :duration-audio-stream duration-audio-stream)
+          (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.ogv")))
+            (expect duration-ms :to-be-weakly-greater-than duration-lower-boundary)
+            (expect duration-ms :to-be-less-than duration-upper-boundary))))))
+  (describe "Get duration in milliseconds of a file with 1 video and 1 audio stream"
+    ;; In this group of test cases, we want the duration of the audio
+    ;; stream to be shorter than the duration of the video stream, so
+    ;; that we can make sure that subed-waveform-ffprobe-duration-ms
+    ;; specifically gets the duration of the audio stream.
+    (let ((duration-video-stream "5")
+          (duration-audio-stream "3")
+          (duration-audio-stream-lower-boundary 3000)
+          (duration-audio-stream-upper-boundary 4000))
+      (it "extension .mkv"
+        (create-sample-media-file-1-audio-stream-1-video-stream
+         :path "/tmp/a.mkv"
+         :duration-video-stream duration-video-stream
+         :duration-audio-stream duration-audio-stream)
+        (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.mkv")))
+          (expect duration-ms :to-be-weakly-greater-than duration-audio-stream-lower-boundary)
+          (expect duration-ms :to-be-less-than duration-audio-stream-upper-boundary)))
+      (it "extension .mp4"
+        (create-sample-media-file-1-audio-stream-1-video-stream
+         :path "/tmp/a.mp4"
+         :duration-video-stream duration-video-stream
+         :duration-audio-stream duration-audio-stream)
+        (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.mp4")))
+          (expect duration-ms :to-be-weakly-greater-than duration-audio-stream-lower-boundary)
+          (expect duration-ms :to-be-less-than duration-audio-stream-upper-boundary)))
+      (it "extension .webm"
+        (create-sample-media-file-1-audio-stream-1-video-stream
+         :path "/tmp/a.webm"
+         :duration-video-stream duration-video-stream
+         :duration-audio-stream duration-audio-stream)
+        (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.webm")))
+          (expect duration-ms :to-be-weakly-greater-than duration-audio-stream-lower-boundary)
+          (expect duration-ms :to-be-less-than duration-audio-stream-upper-boundary)))
+      (it "extension .avi"
+        (create-sample-media-file-1-audio-stream-1-video-stream
+         :path "/tmp/a.avi"
+         :duration-video-stream duration-video-stream
+         :duration-audio-stream duration-audio-stream)
+        (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.avi")))
+          (expect duration-ms :to-be-weakly-greater-than duration-audio-stream-lower-boundary)
+          (expect duration-ms :to-be-less-than duration-audio-stream-upper-boundary)))
+      (it "extension .ts"
+        (create-sample-media-file-1-audio-stream-1-video-stream
+         :path "/tmp/a.ts"
+         :duration-video-stream duration-video-stream
+         :duration-audio-stream duration-audio-stream)
+        (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.ts")))
+          (expect duration-ms :to-be-weakly-greater-than duration-audio-stream-lower-boundary)
+          (expect duration-ms :to-be-less-than duration-audio-stream-upper-boundary)))
+      (it "extension .ogv"
+        (create-sample-media-file-1-audio-stream-1-video-stream
+         :path "/tmp/a.ogv"
+         :duration-video-stream duration-video-stream
+         :duration-audio-stream duration-audio-stream)
+        (let ((duration-ms (subed-waveform-ffprobe-duration-ms "/tmp/a.ogv")))
+          (expect duration-ms :to-be-weakly-greater-than duration-audio-stream-lower-boundary)
+          (expect duration-ms :to-be-less-than duration-audio-stream-upper-boundary))))))