Subtitle timeline align (#47)

subtitle alignment

.github/workflows/publish.yaml

@@ -31,7 +31,12 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '11'
 
       - name: Install hunspell
         run:  sudo apt-get install -y hunspell libhunspell-dev

.gitignore

@@ -134,3 +134,4 @@ dmypy.json
 
 fixtures/*.srt
 .vscode/
+.DS_Store

Dockerfile

@@ -41,7 +41,7 @@ COPY requirements.txt /tmp/
 # vosk doesn't install libatomic1 dep on aarch64
 RUN apt-get -q update && \
     apt-get install -y software-properties-common && \
-    apt-get install -qy zsh ffmpeg x264 x265 imagemagick vainfo curl python3 python3-pip python3-dev cron anacron sshfs vim-tiny mkvtoolnix unzip logrotate jq less \
+    apt-get install -qy zsh ffmpeg x264 x265 imagemagick vainfo curl python3 python3-pip python3-dev cron anacron sshfs vim-tiny mkvtoolnix unzip logrotate jq less default-jre \
     mono-runtime libmono-system-windows-forms4.0-cil libmono-system-net-http-webrequest4.0-cil mono-devel libhunspell-dev tesseract-ocr-eng xserver-xorg-video-dummy libgtk2.0-0 \
     libargtable2-0 libavformat59 libsdl1.2-compat libatomic1 &&\
     pip --no-input install --break-system-packages --compile --ignore-installed -r /tmp/requirements.txt && \
@@ -55,6 +55,7 @@ RUN apt-get -q update && \
     unzip -d /root/.cache/vosk /tmp/vosk-model-en-us-0.22-lgraph.zip &&\
     curl -o /tmp/vosk-model-small-es-0.22.zip -L --silent --fail https://alphacephei.com/vosk/models/vosk-model-small-es-0.22.zip &&\
     unzip -d /root/.cache/vosk /tmp/vosk-model-small-es-0.22.zip &&\
+    python3 -c "import language_tool_python; tool = language_tool_python.LanguageTool('en')" &&\
     rm -rf /tmp/*
 
 # It appears Ubuntu does not include tesseract models for all OCR engines

dvrprocess/common/__init__.py

@@ -791,6 +791,10 @@ def s_to_ts(t: float) -> str:
     return f"{hour:02d}:{minute:02d}:{second:06.3f}"
 
 
+def ms_to_ts(t: float) -> str:
+    return s_to_ts(t / 1000.0)
+
+
 def write_mkv_tags(tags, filepath) -> None:
     """
     Writes the dictionary to an XML file for use my mkvpropedit.
@@ -920,7 +924,7 @@ def setup_debugging():
 
 
 def setup_logging(level=logging.INFO):
-    logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s %(message)s', level=level)
+    logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s:%(lineno)d %(message)s', level=level, force=True)
 
 
 def setup_cli(level=logging.INFO):
@@ -947,8 +951,7 @@ def __init__(self, func):
         self.progress_queue = progress.setup_parent_progress()
 
     def __call__(self, *args, **kwargs):
-        progress.setup_subprocess_progress(self.progress_queue)
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        progress.setup_subprocess_progress(self.progress_queue, self.rootLogLevel)
         stdout = StreamCapture('stdout', logger, logging.INFO)
         stderr = StreamCapture('stderr', logger, logging.ERROR)
         try:

dvrprocess/common/censor_list.txt

@@ -12,6 +12,7 @@ oh jesus
 oh gods
 oh god almighty
 oh god in heaven
+god in heaven
 oh my god
 oh my gad
 oh my gods
@@ -125,8 +126,8 @@ dang you
 oh dang
 oh dang it
 bitch
-bitchs
-bitches
+bitchy
+bitche?s
 bitched
 son[\s-]of[\s-]a[\s-]bitch
 sons[\s-]of[\s-]bitches
@@ -212,6 +213,7 @@ that mother[!.-]+
 you mother[!.-]+
 holy mother\w*
 holy mother of god
+bad[\s-]?ass mother
 freakin
 freaking
 friggin
@@ -448,6 +450,7 @@ g[ia]ve [\w\s]* the finger
 sexy on
 skank
 slut
+slutt?y
 mo[\s-]?fos?
 douche[\w-]*
 dick[\s-]?heads?
@@ -458,6 +461,8 @@ dicking
 dicked
 mean dick
 count for dick
+the dick
+private dick
 pecker
 his pecker
 my pecker

dvrprocess/common/constants.py

@@ -20,6 +20,8 @@
 K_COMSKIP_SKIP = 'COMSKIP_SKIP'
 K_AUDIO_TO_TEXT_VERSION = 'AUDIO2TEXT_VERSION'
 K_AUDIO_TO_TEXT_FILTER = 'AUDIO2TEXT_FILTER'
+K_AUDIO_TO_TEXT_SUBTITLE_VERSION = 'AUDIO2TEXT_SUBTITLE'
+K_SUB_ALIGNMENT_STATS = 'sub_alignment_stats'
 K_MEDIA_TITLE = 'title'
 K_MEDIA_PROCESSOR = 'processor'
 V_MEDIA_PROCESSOR = 'media-hare, https://github.com/double16/media-hare'

dvrprocess/common/proc_invoker.py

@@ -173,7 +173,7 @@ def required(self) -> bool:
         return self._required
 
     def install(self) -> bool:
-        logger.error('Install "%s" %s and try again.', self.command_basename, ' '.join(self.version_target))
+        logger.error('Install "%s" %s and try again.', self.command_basename, ' '.join(self.version_target or []))
         return False
 
     def present(self) -> bool:

dvrprocess/common/progress.py

@@ -1,6 +1,7 @@
 import logging
 import multiprocessing
 import os
+import signal
 import threading
 import time
 from math import ceil
@@ -306,7 +307,7 @@ def __init__(self, task: str, position: int, msg: Union[None, str] = None, start
 
     def apply(self):
         try:
-            _PROGRESS_BY_TASK.get(self.task).progress(self.position, self.msg, self.start, self.end)
+            _PROGRESS_BY_TASK[self.task].progress(self.position, self.msg, self.start, self.end)
         except KeyError:
             pass
 
@@ -321,8 +322,9 @@ def __init__(self, task: str, msg: Union[str, None] = None):
 
     def apply(self):
         try:
-            _PROGRESS_BY_TASK.get(self.task).stop(self.msg)
-            _PROGRESS_BY_TASK.pop(self.task)
+            p = _PROGRESS_BY_TASK.pop(self.task)
+            if p:
+                p.stop(self.msg)
         except KeyError:
             pass
 
@@ -355,7 +357,7 @@ def __init__(self, name: str, value: float):
 
     def apply(self):
         try:
-            _GAUGE_BY_NAME.get(self.name).value(self.value)
+            _GAUGE_BY_NAME[self.name].value(self.value)
         except KeyError:
             pass
 
@@ -429,7 +431,7 @@ def _progress_queue_feed(q: Queue):
     while True:
         try:
             m = q.get(True)
-        except (ValueError, EOFError):
+        except (ValueError, EOFError, BrokenPipeError):
             # queue is closed
             return
         try:
@@ -455,16 +457,20 @@ def setup_parent_progress() -> Queue:
 _subprocess_progress_configured = False
 
 
-def setup_subprocess_progress(progress_queue: Queue):
+def setup_subprocess_progress(progress_queue: Queue, level: int):
     """
     Setup this process to send progress to the parent process.
     """
     global _subprocess_progress_configured
     if _subprocess_progress_configured:
         return
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
     set_progress_reporter(SubprocessProgressReporter(progress_queue))
+    for h in logging.root.handlers[:]:
+        logging.root.removeHandler(h)
+        h.close()
     logging.root.addHandler(SubprocessLogHandler(progress_queue))
-    logging.root.setLevel(logging.DEBUG)
+    logging.root.setLevel(level)
     _subprocess_progress_configured = True
 
 

dvrprocess/common/stop_list.txt

@@ -1,34 +1,23 @@
-genital
-genitals
+genitals?
 penis
 penises
-vagina
-vaginas
+vaginas?
 pussy
 pussies
-cunt
-cunts
-twat
-twats
-nutsack
-nutsacks
-nut sack
-nut sacks
-nut-sack
-nut-sacks
-ball[\s-]sack
-ball[\s-]sacks
+cunts?
+twats?
+nut[\s-]?sacks?
+ball[\s-]?sacks?
 testicles?
 gonads?
 scheiße
 fondle
 tryst
 skimpy
-thong
-orgasm
-orgasms
-orgi
-orgis
+thongs?
+orgasms?
+orgasmic
+orgis?
 orgies
 semen
 ejaculate
@@ -39,16 +28,12 @@ butt[\s-]sex
 booty[\s-]sex
 ass[\s-]sex
 gay[\s-]sex
-cock
-cocks
+cocks?
 cock's
 cock’s
-cocksucker
-cocksuckers
 cocksucker's
 cocksucker’s
-cock[\s-]sucker
-cock[\s-]suckers
+cock[\s-]?suckers?
 a dick
 real dick
 stupid dick
@@ -78,18 +63,17 @@ tittys
 titty's
 tities
 titties
-boob
-boobs
+boobs?
 boob's
 boob’s
 boobies
 breasts
 kinky
 horny
-butt[\s-]fuck
-butt[\s-]fuckers?
-ass[\s-]fuck
-ass[\s-]fuckers?
+butt[\s-]?fuck
+butt[\s-]?fuckers?
+ass[\s-]?fuck
+ass[\s-]?fuckers?
 masturbate
 masturbates
 masturbated
@@ -151,11 +135,11 @@ slept with
 slept together
 boner
 nude
-hooker
+hookers?
 hooking
 wear a rubber
 use a rubber
-condom
+condoms?
 conjugal
 fluffer
 cherry popped
@@ -171,18 +155,18 @@ skinny[-\s]dip\w*
 sucking the sack
 junk in \w+ trunk
 hook[\s-]?up site
-pimp
+pimps?
 pimping
 scrotums?
 canoodling
 canoodle
-stripper pole
+stripper poles?
 she .* stripper
 he .* stripper
 tongue bath
 camel[\s-]?toe
 camel[\s-]?knuckle
-dildo
+dildos?
 cajones?
 fore[\s-]?play
 nice tail
@@ -191,8 +175,12 @@ after some tail
 some \w+ tail
 piece of tail
 whore[\s-]?houses?
+brothel
+trade sex
+swap sex
 the clap
 cleavage
 g[eo]t laid
 negligee
 nudists?
+felch\w*