Release pipeline

.github/workflows/package_for_release.yml

@@ -0,0 +1,51 @@
+name: package for release
+on:
+    workflow_dispatch: {}
+    pull_request:
+        types: [opened, synchronize]
+
+jobs:
+    linux:
+        runs-on: ubuntu-latest
+        container:
+            image: ubuntu:mantic-20230712
+        steps:
+            - name: 🛠 Install system dependencies
+              run: |
+                  apt-get update -y -qq
+                  apt-get install -y libegl1-mesa-dev libgl1-mesa-dri libxcb-xfixes0-dev ffmpeg libavcodec-dev libavformat-dev libavfilter-dev libavdevice-dev
+                  # TMP until we need to build in docker container
+                  apt-get install -y curl build-essential curl pkg-config libssl-dev libclang-dev git libnss3 libatk1.0-0 libatk-bridge2.0-0 libgdk-pixbuf2.0-0 libgtk-3-0
+            - name: 🔧 Install the rust toolchain
+              uses: dtolnay/rust-toolchain@stable
+
+            - name: 📥 Checkout repo
+              uses: actions/checkout@v3
+
+            - name: 📦 Package
+              run: cargo run --bin package_for_release --features standalone
+
+            - uses: actions/upload-artifact@v3
+              with:
+                name: video_compositor_linux_x86_64.tar.gz
+                path: video_compositor_linux_x86_64.tar.gz
+
+    macos:
+        runs-on: macos-latest
+        steps:
+            - name: 🛠 Install system dependencies
+              run: brew install ffmpeg
+
+            - name: 🔧 Install the rust toolchain
+              uses: dtolnay/rust-toolchain@stable
+
+            - name: 📥 Checkout repo
+              uses: actions/checkout@v3
+
+            - name: 📦 Package
+              run: cargo run --bin package_for_release --features standalone
+
+            - uses: actions/upload-artifact@v3
+              with:
+                name: video_compositor_darwin_x86_64.app.tar.gz
+                path: video_compositor_darwin_x86_64.app.tar.gz

.github/workflows/shellcheck.yml

@@ -0,0 +1,28 @@
+name: shellcheck
+on:
+    push:
+        branches: [master]
+        paths:
+            - "scripts/*"
+            - .github/workflows/shellcheck.yml
+    pull_request:
+        types: [opened, synchronize]
+        paths:
+            - "scripts/*"
+            - .github/workflows/shellcheck.yml
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+jobs:
+    dockerfile:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v2
+
+            - name: shellcheck
+              run: |
+                shellcheck ./scripts/release.sh ./scripts/compositor_runtime_wrapper.sh

.gitignore

@@ -26,3 +26,9 @@ target/
 
 # Generated during runtime
 /shmem
+
+/video_compositor.app
+/video_compositor
+/video_compositor_linux_x86_64.tar.gz
+/video_compositor_darwin_aarch64.app.tar.gz
+/video_compositor_darwin_x86_64.app.tar.gz

Cargo.toml

@@ -14,6 +14,10 @@ members = [
 ]
 resolver = "2"
 
+[features]
+default = []
+standalone = ["compositor_chromium/standalone"]
+
 [workspace.dependencies]
 bytes = "1.4.0"
 env_logger = "0.10.0"
@@ -27,6 +31,7 @@ reqwest = { version = "0.11.18", features = ["blocking", "json"] }
 signal-hook = "0.3.15"
 shared_memory = "0.12.4"
 ffmpeg-next = "6.0.0"
+anyhow = "1.0.71"
 
 [dependencies]
 compositor_render = { path = "compositor_render" }
@@ -35,7 +40,7 @@ compositor_pipeline = { path = "compositor_pipeline" }
 compositor_chromium = { path = "compositor_chromium" }
 serde = { workspace = true }
 serde_json = { workspace = true }
-anyhow = "1.0.71"
+anyhow = { workspace = true }
 bytes = { workspace = true }
 tiny_http = "0.12.0"
 ffmpeg-next = { workspace = true }
@@ -45,6 +50,7 @@ log = { workspace = true }
 signal-hook = { workspace = true }
 shared_memory = { workspace = true }
 lazy_static = "1.4.0"
+fs_extra = "1.3.0"
 
 [dev-dependencies]
 reqwest = { workspace = true }

RELEASE.md

@@ -0,0 +1,24 @@
+# Release
+
+To release a new compositor version:
+
+- Go to `Actions` -> [`package for release`](https://github.com/membraneframework/video_compositor/actions/workflows/release.yml) -> Dispatch new workflow on a master branch.
+- Wait for a build to finish.
+- Run `gh run list --workflow "package for release"` and find ID of workflow run you just run.
+- Run `WORKFLOW_RUN_ID={WORKFLOW_RUN_ID} RELEASE_TAG={VERSION} ./scripts/release.sh` where `WORKFLOW_RUN_ID` is an ID from the previous step, and `VERSION` has a format `v{major}.{minor}.{patch}`. e.g. `WORKFLOW_RUN_ID=6302155380 RELEASE_TAG=v1.2.3 ./scripts/release.sh `
+
+
+### Temporary workaround for macOS M1 binaries
+
+Currently we do not have a CI to build for macOS M1, so for now compositor releases are run from a developer device.
+
+To publish binaries for M1 devices, first follow instructions above for other platform. After GitHub release is created you can add binaries for M1 by running bellow command on M1 mac.
+
+```bash
+RELEASE_TAG={VERSION} cargo run --bin package_for_release --features standalone
+```
+
+e.g.
+```bash
+RELEASE_TAG=v1.2.3 cargo run --bin package_for_release --features standalone
+```

compositor_chromium/Cargo.toml

@@ -5,10 +5,15 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = []
+standalone = []
+
 [dependencies]
 chromium_sys = { path = "chromium_sys" }
 compositor_common = { path = "../compositor_common" }
 fs_extra = "1.3.0"
 thiserror = { workspace = true }
 widestring = "1.0.2"
 log = { workspace = true }
+anyhow = { workspace = true }

compositor_chromium/chromium_sys/build.rs

@@ -1,6 +1,7 @@
 use anyhow::{Context, Result};
 use bindgen::callbacks::ParseCallbacks;
 use fs_extra::dir::{self, CopyOptions};
+use reqwest::StatusCode;
 use std::{
     env, fs,
     path::{Path, PathBuf},
@@ -124,27 +125,31 @@ impl ParseCallbacks for RemoveCommentsCallback {
     }
 }
 
-fn download_cef(cef_root_path: &Path) -> Result<()> {
-    let platform = if cfg!(target_os = "macos") {
+fn cef_url() -> &'static str {
+    if cfg!(target_os = "macos") {
         if cfg!(target_arch = "aarch64") {
-            "macosarm64"
-        } else {
-            "macosx64"
+            return "https://cef-builds.spotifycdn.com/cef_binary_117.1.4%2Bga26f38b%2Bchromium-117.0.5938.92_macosarm64_minimal.tar.bz2";
+        } else if cfg!(target_arch = "x86_64") {
+            return "https://cef-builds.spotifycdn.com/cef_binary_115.3.11%2Bga61da9b%2Bchromium-115.0.5790.114_macosx64_minimal.tar.bz2";
         }
     } else if cfg!(target_os = "linux") {
-        "linux64"
-    } else {
-        panic!("Unsupported platform");
+        return "https://cef-builds.spotifycdn.com/cef_binary_117.1.4%2Bga26f38b%2Bchromium-117.0.5938.92_linux64_minimal.tar.bz2";
     };
+    panic!("Unsupported platform");
+}
 
+fn download_cef(cef_root_path: &Path) -> Result<()> {
+    let url = cef_url();
     let download_path = cef_root_path
         .parent()
         .context("Failed to retrieve CEF_ROOT parent directory")?;
-    let url = format!("https://cef-builds.spotifycdn.com/cef_binary_117.1.4%2Bga26f38b%2Bchromium-117.0.5938.92_{platform}_minimal.tar.bz2");
     let client = reqwest::blocking::ClientBuilder::new()
         .timeout(Duration::from_secs(2 * 60))
         .build()?;
     let resp = client.get(url).send()?;
+    if resp.status() != StatusCode::OK {
+        panic!("Request to {} failed. Status code: {}", url, resp.status());
+    }
 
     let archive_name = "cef.tar.bz2";
     let content = resp.bytes()?;

compositor_chromium/examples/simple.rs

@@ -86,7 +86,7 @@ fn main() {
         .unwrap()
         .join("..");
 
-    if cef::bundle_app(target_path).is_err() {
+    if cef::bundle_for_development(target_path).is_err() {
         panic!("Build process helper first: cargo build --bin process_helper");
     }
 

compositor_chromium/src/settings.rs

@@ -1,7 +1,11 @@
-use std::{env, os::raw::c_int, path::PathBuf};
+use std::{env, os::raw::c_int};
+
+use chromium_sys::_cef_string_utf16_t;
 
 use crate::cef_string::CefString;
 
+pub const PROCESS_HELPER_PATH_ENV: &str = "MEMBRANE_VIDEO_COMPOSITOR_PROCESS_HELPER_PATH";
+
 /// Main process settings
 #[derive(Default)]
 pub struct Settings {
@@ -18,38 +22,14 @@ pub struct Settings {
 
 impl Settings {
     pub fn into_raw(self) -> chromium_sys::cef_settings_t {
-        let current_exe = env::current_exe().unwrap();
-        let current_dir = current_exe.parent().unwrap();
-
-        let main_bundle_path = if cfg!(target_os = "linux") {
-            String::new()
-        } else {
-            PathBuf::from(current_dir)
-                .join("video_compositor.app")
-                .display()
-                .to_string()
-        };
-
-        let browser_subprocess_path = if cfg!(target_os = "linux") {
-            current_dir.join("process_helper").display().to_string()
-        } else {
-            PathBuf::from(&main_bundle_path)
-                .join("Contents")
-                .join("Frameworks")
-                .join("video_compositor Helper.app")
-                .join("Contents")
-                .join("MacOS")
-                .join("video_compositor Helper")
-                .display()
-                .to_string()
-        };
+        let (main_path, helper_path) = executables_paths();
 
         chromium_sys::cef_settings_t {
             size: std::mem::size_of::<chromium_sys::cef_settings_t>(),
             no_sandbox: true as c_int,
-            browser_subprocess_path: CefString::new_raw(browser_subprocess_path),
+            browser_subprocess_path: helper_path,
             framework_dir_path: CefString::empty_raw(),
-            main_bundle_path: CefString::new_raw(main_bundle_path),
+            main_bundle_path: main_path,
             chrome_runtime: false as c_int,
             multi_threaded_message_loop: self.multi_threaded_message_loop as c_int,
             external_message_pump: self.external_message_pump as c_int,
@@ -64,6 +44,7 @@ impl Settings {
             locale: CefString::empty_raw(),
             log_file: CefString::empty_raw(),
             log_severity: self.log_severity as u32,
+            #[cfg(not(all(target_os = "macos", target_arch = "x86_64")))]
             log_items: chromium_sys::cef_log_items_t_LOG_ITEMS_DEFAULT,
             javascript_flags: CefString::empty_raw(),
             resources_dir_path: CefString::empty_raw(),
@@ -95,3 +76,49 @@ impl Default for LogSeverity {
         Self::Default
     }
 }
+
+#[cfg(target_os = "linux")]
+fn executables_paths() -> (_cef_string_utf16_t, _cef_string_utf16_t) {
+    let browser_subprocess_path = env::var(PROCESS_HELPER_PATH_ENV).unwrap_or_else(|_| {
+        let current_exe = env::current_exe().unwrap();
+        let current_dir = current_exe.parent().unwrap();
+        current_dir.join("process_helper").display().to_string()
+    });
+
+    (
+        CefString::empty_raw(),
+        CefString::new_raw(browser_subprocess_path),
+    )
+}
+
+#[cfg(target_os = "macos")]
+fn executables_paths() -> (_cef_string_utf16_t, _cef_string_utf16_t) {
+    use std::path::PathBuf;
+
+    if cfg!(feature = "standalone") {
+        return (CefString::empty_raw(), CefString::empty_raw());
+    }
+
+    let current_exe = env::current_exe().unwrap();
+    let current_dir = current_exe.parent().unwrap();
+
+    let main_bundle_path = PathBuf::from(current_dir)
+        .join("video_compositor.app")
+        .display()
+        .to_string();
+
+    let browser_subprocess_path = PathBuf::from(&main_bundle_path)
+        .join("Contents")
+        .join("Frameworks")
+        .join("video_compositor Helper.app")
+        .join("Contents")
+        .join("MacOS")
+        .join("video_compositor Helper")
+        .display()
+        .to_string();
+
+    (
+        CefString::new_raw(main_bundle_path),
+        CefString::new_raw(browser_subprocess_path),
+    )
+}