Chrome sandbox (#414)

* add sandbox build target * ＼(＾O＾)／ * clean up * add logs * try 4755 * more testing * clean up debugging * use mkdirall
livekit · Jul 6, 2023 · 331c179 · 331c179
1 parent 717af1b
commit 331c179
Show file tree

Hide file tree

Showing 11 changed files with 72 additions and 55 deletions.
diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml
@@ -53,9 +53,9 @@ jobs:
           docker run --rm \
             --network host \
             --security-opt seccomp=unconfined \
+            --cap-add=SYS_ADMIN \
             -e GITHUB_WORKFLOW=1 \
             -e EGRESS_CONFIG_STRING="$(echo ${{ secrets.EGRESS_CONFIG_STRING }} | base64 -d)" \
             -e S3_UPLOAD="$(echo ${{ secrets.S3_UPLOAD }} | base64 -d)" \
             -e GCP_UPLOAD="$(echo ${{ secrets.GCP_UPLOAD }} | base64 -d)" \
-            -v /workspace/test:/out \
             egress-test
diff --git a/build/chrome/README.md b/build/chrome/README.md
@@ -74,10 +74,12 @@ cd src
 ./build/linux/sysroot_scripts/install-sysroot.py --arch=arm64
 gclient runhooks
 gn gen out/default --args='target_cpu="arm64" proprietary_codecs=true ffmpeg_branding="Chrome" enable_nacl=false is_debug=false symbol_level=0 v8_symbol_level=0 dcheck_always_on=false is_official_build=true'
-autoninja -C out/default chrome
+autoninja -C out/default chrome chrome_sandbox
+cd out/default
 zip arm64.zip \
   chrome \
   chrome-wrapper \
+  chrome_sandbox \
   chrome_100_percent.pak \
   chrome_200_percent.pak \
   chrome_crashpad_handler \
@@ -97,5 +99,6 @@ exit
 ```
 ```shell
 scp root@$CHROME_BUILDER:/home/chrome/chromium/src/out/default/arm64.zip ~/livekit/egress/build/chrome/arm64.zip
-unzip ~/livekit/egress/build/chrome/arm64.zip && rm ~/livekit/egress/build/chrome/arm64.zip
+cd ~/livekit/egress/build/chrome
+mkdir arm64 && unzip arm64.zip -d arm64 && rm arm64.zip
 ```
diff --git a/build/chrome/install-chrome b/build/chrome/install-chrome
@@ -3,7 +3,6 @@ set -euxo pipefail
 
 if [ "$1" = "linux/arm64" ]
 then
-  mv /chrome-installer/arm64/ /chrome
   apt-get update
   apt-get install -y \
     ca-certificates \
@@ -37,6 +36,10 @@ then
     libxrender1 \
     libxss1 \
     libxtst6
+  mv /chrome-installer/arm64/ /chrome
+  cp /chrome/chrome_sandbox /usr/local/sbin/chrome-devel-sandbox
+  chown root:root /usr/local/sbin/chrome-devel-sandbox
+  chmod 4755 /usr/local/sbin/chrome-devel-sandbox
 else
   wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
   apt-get install -y ./google-chrome-stable_current_amd64.deb
@@ -48,3 +51,5 @@ unzip chromedriver_linux64.zip
 chmod +x chromedriver
 mv -f chromedriver /usr/local/bin/chromedriver
 rm chromedriver_linux64.zip
+
+rm -rf /chrome-installer
diff --git a/build/egress/Dockerfile b/build/egress/Dockerfile
@@ -44,22 +44,24 @@ RUN apt-get update && \
 
 # install chrome
 COPY --from=livekit/chrome-installer:latest /chrome-installer /chrome-installer
-RUN /chrome-installer/install-chrome "$TARGETPLATFORM" && \
-    rm -rf /chrome-installer
+RUN /chrome-installer/install-chrome "$TARGETPLATFORM"
 
-# setup
-RUN adduser root pulse-access && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    mkdir -pv ~/.cache/xdgr
+# clean up
+RUN rm -rf /var/lib/apt/lists/*
 
-# update env
-ENV PATH=${PATH}:/chrome
-ENV XDG_RUNTIME_DIR=$PATH:~/.cache/xdgr
+# create egress user
+RUN useradd -ms /bin/bash -g root -G sudo,pulse,pulse-access egress
+RUN mkdir -pv home/egress/.cache/xdgr && \
+    chown -R egress /home/egress && \
+    chmod 777 /home/egress
 
-# egress
+# copy files
 COPY --from=0 /workspace/egress /bin/
+COPY build/egress/entrypoint.sh /
 
 # run
-COPY build/egress/entrypoint.sh /
+USER egress
+ENV PATH=${PATH}:/chrome
+ENV XDG_RUNTIME_DIR=/home/egress/.cache/xdgr
+ENV CHROME_DEVEL_SANDBOX=/usr/local/sbin/chrome-devel-sandbox
 ENTRYPOINT ["/entrypoint.sh"]
diff --git a/build/egress/entrypoint.sh b/build/egress/entrypoint.sh
@@ -3,11 +3,9 @@ set -euxo pipefail
 
 export TMPDIR=/tmp/lkegress
 
-# Cleanup to be "stateless" on startup, otherwise pulseaudio daemon can't start again
-rm -rf /var/run/pulse /var/lib/pulse /root/.config/pulse
-
-# Start pulseaudio as system wide daemon; for debugging it helps to start in non-daemon mode
-pulseaudio -D --verbose --exit-idle-time=-1 --system --disallow-exit
+# Start pulseaudio
+rm -rf /var/run/pulse /var/lib/pulse /home/egress/.config/pulse
+pulseaudio -D --verbose --exit-idle-time=-1 --disallow-exit
 
 # cleanup old temporary files
 if ! [ -z $TMPDIR ]; then

diff --git a/build/test/Dockerfile b/build/test/Dockerfile
@@ -49,37 +49,35 @@ RUN apt-get update && \
         gstreamer1.0-plugins-base-
 
 # install chrome
-COPY --from=livekit/chrome-installer:latest /chrome-installer /chrome-installer
-RUN /chrome-installer/install-chrome "$TARGETPLATFORM" && \
-    rm -rf /chrome-installer
+COPY --from=livekit/chrome-installer:117.0.5874.0 /chrome-installer /chrome-installer
+RUN /chrome-installer/install-chrome "$TARGETPLATFORM"
+
+# clean up
+RUN rm -rf /var/lib/apt/lists/*
 
 # install rtsp server
 RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCH=arm64v8; else ARCH=amd64; fi && \
     wget https://github.com/aler9/rtsp-simple-server/releases/download/v0.17.6/rtsp-simple-server_v0.17.6_linux_${ARCH}.tar.gz && \
     tar -zxvf rtsp-simple-server_v0.17.6_linux_${ARCH}.tar.gz && \
     rm rtsp-simple-server_v0.17.6_linux_${ARCH}.tar.gz
 
-# add root user to group for pulseaudio access
-RUN adduser root pulse-access
-
-# clean up
-RUN rm -rf /var/lib/apt/lists/* && \
-    rm -rf /var/run/pulse /var/lib/pulse /root/.config/pulse && \
-    mkdir -pv ~/.cache/xdgr
-
-# update env
-ENV PATH=${PATH}:/chrome
-ENV XDG_RUNTIME_DIR=$PATH:~/.cache/xdgr
-ENV RTSP_LOGDESTINATIONS=file
+# create egress user
+RUN useradd -ms /bin/bash -g root -G sudo,pulse,pulse-access egress
+RUN mkdir -pv home/egress/.cache/xdgr && \
+    chown -R egress /home/egress
 
+# copy files
 COPY test/ /workspace/test/
-
-# egress
 COPY --from=0 /workspace/egress /bin/
-
-# test
 COPY --from=0 /workspace/test.test .
-
-# run
 COPY build/test/entrypoint.sh .
+
+# create output directory
+RUN mkdir -p /out/output && chown egress /out/output
+
+# run tests
+USER egress
+ENV PATH=${PATH}:/chrome
+ENV XDG_RUNTIME_DIR=/home/egress/.cache/xdgr
+ENV CHROME_DEVEL_SANDBOX=/usr/local/sbin/chrome-devel-sandbox
 ENTRYPOINT ["./entrypoint.sh"]
diff --git a/build/test/entrypoint.sh b/build/test/entrypoint.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -exo pipefail
 
-# Start pulseaudio as system wide daemon; for debugging it helps to start in non-daemon mode
-pulseaudio -D --verbose --exit-idle-time=-1 --system --disallow-exit
+# Start pulseaudio
+rm -rf /var/run/pulse /var/lib/pulse /home/egress/.config/pulse
+pulseaudio -D --verbose --exit-idle-time=-1 --disallow-exit
 
 # Run RTSP server
-./rtsp-simple-server &
+./rtsp-simple-server > /dev/null 2>&1 &
 
 # Run tests
 if [[ -z ${GITHUB_WORKFLOW+x} ]]; then

diff --git a/magefile.go b/magefile.go
@@ -13,7 +13,12 @@ import (
 	"github.com/livekit/mageutil"
 )
 
-const gstVersion = "1.22.4"
+const (
+	gstVersion      = "1.22.4"
+	chromiumVersion = "117.0.5874.0"
+	dockerBuild     = "docker build"
+	dockerBuildX    = "docker buildx build --push --platform linux/amd64,linux/arm64"
+)
 
 type packageInfo struct {
 	Dir string
@@ -93,13 +98,13 @@ func Integration(configFile string) error {
 
 	return mageutil.Run(context.Background(),
 		"docker build -t egress-test -f build/test/Dockerfile .",
-		fmt.Sprintf("docker run --rm -e EGRESS_CONFIG_FILE=%s -v %s/test:/out egress-test", configFile, dir),
+		fmt.Sprintf("docker run --rm -e EGRESS_CONFIG_FILE=%s -v %s/test:/out --cap-add=SYS_ADMIN egress-test", configFile, dir),
 	)
 }
 
 func Build() error {
 	return mageutil.Run(context.Background(),
-		"docker pull livekit/chrome-installer",
+		fmt.Sprintf("docker pull livekit/chrome-installer:%s", chromiumVersion),
 		fmt.Sprintf("docker pull livekit/gstreamer:%s-dev", gstVersion),
 		"docker pull livekit/egress-templates",
 		"docker build -t livekit/egress:latest -f build/egress/Dockerfile .",
@@ -116,7 +121,10 @@ func BuildChrome() error {
 func PublishChrome() error {
 	return mageutil.Run(context.Background(),
 		"docker pull ubuntu:22.04",
-		"docker buildx build --push --platform linux/amd64,linux/arm64 -t livekit/chrome-installer ./build/chrome",
+		fmt.Sprintf(
+			"%s -t livekit/chrome-installer:%s ./build/chrome",
+			dockerBuildX, chromiumVersion,
+		),
 	)
 }
 
@@ -128,11 +136,11 @@ func BuildTemplate() error {
 }
 
 func BuildGStreamer() error {
-	return buildGstreamer("docker build")
+	return buildGstreamer(dockerBuild)
 }
 
 func PublishGStreamer() error {
-	return buildGstreamer("docker buildx build --push --platform linux/amd64,linux/arm64")
+	return buildGstreamer(dockerBuildX)
 }
 
 func buildGstreamer(cmd string) error {

diff --git a/pkg/config/service.go b/pkg/config/service.go
@@ -80,6 +80,9 @@ func NewServiceConfig(confString string) (*ServiceConfig, error) {
 	if conf.LocalOutputDirectory == "." {
 		conf.LocalOutputDirectory = os.TempDir()
 	}
+	if err := os.MkdirAll(conf.LocalOutputDirectory, 0755); err != nil {
+		return nil, err
+	}
 
 	if err := conf.initLogger("nodeID", conf.NodeID, "clusterID", conf.ClusterID); err != nil {
 		return nil, err

diff --git a/pkg/pipeline/source/web.go b/pkg/pipeline/source/web.go
@@ -141,7 +141,7 @@ func (s *WebSource) launchXvfb(ctx context.Context, p *config.PipelineConfig) er
 
 	dims := fmt.Sprintf("%dx%dx%d", p.Width, p.Height, p.Depth)
 	logger.Debugw("launching xvfb", "display", p.Display, "dims", dims)
-	xvfb := exec.Command("Xvfb", p.Display, "-screen", "0", dims, "-ac", "-nolisten", "tcp")
+	xvfb := exec.Command("Xvfb", p.Display, "-screen", "0", dims, "-ac", "-nolisten", "tcp", "-nolisten", "unix")
 	xvfb.Stderr = &errorLogger{cmd: "xvfb"}
 	if err := xvfb.Start(); err != nil {
 		return errors.Fatal(errors.ErrProcessStartFailed(err))
@@ -190,7 +190,7 @@ func (s *WebSource) launchChrome(ctx context.Context, p *config.PipelineConfig,
 		chromedp.Flag("disable-default-apps", true),
 		chromedp.Flag("disable-dev-shm-usage", true),
 		chromedp.Flag("disable-extensions", true),
-		chromedp.Flag("disable-features", "site-per-process,TranslateUI,BlinkGenPropertyTrees"),
+		chromedp.Flag("disable-features", "AudioServiceOutOfProcess,site-per-process,TranslateUI,BlinkGenPropertyTrees"),
 		chromedp.Flag("disable-hang-monitor", true),
 		chromedp.Flag("disable-ipc-flooding-protection", true),
 		chromedp.Flag("disable-popup-blocking", true),
@@ -204,7 +204,6 @@ func (s *WebSource) launchChrome(ctx context.Context, p *config.PipelineConfig,
 		chromedp.Flag("use-mock-keychain", true),
 
 		// custom args
-		// TODO: chromedp.Flag("no-sandbox", false),
 		chromedp.Flag("kiosk", true),
 		chromedp.Flag("enable-automation", false),
 		chromedp.Flag("autoplay-policy", "no-user-gesture-required"),

diff --git a/test/integration.go b/test/integration.go
@@ -31,7 +31,7 @@ const (
 	redactedBadUrl1 = "rtmp://sfo.contribute.live-video.net/app/{f...1}"
 	badStreamUrl2   = "rtmp://localhost:1936/live/stream"
 	redactedBadUrl2 = "rtmp://localhost:1936/live/{st...am}"
-	webUrl          = "https://www.youtube.com/watch?v=wjQq0nSGS28&t=5205s"
+	webUrl          = "https://videoplayer-2k23.vercel.app/videos/eminem"
 )
 
 var (