diff --git a/.github/workflows/apk.yaml b/.github/workflows/apk.yaml index a03420bc2..180b0c14f 100644 --- a/.github/workflows/apk.yaml +++ b/.github/workflows/apk.yaml @@ -38,6 +38,7 @@ jobs: shell: bash run: | export ANDROID_NDK=$ANDROID_NDK_LATEST_HOME + ./build-apk-vad.sh ./build-apk-two-pass.sh ./build-apk.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index b111d5354..5c0a7c19b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR) project(sherpa-onnx) -set(SHERPA_ONNX_VERSION "1.7.18") +set(SHERPA_ONNX_VERSION "1.7.19") # Disable warning about # diff --git a/android/README.md b/android/README.md new file mode 100644 index 000000000..4e6e3c99c --- /dev/null +++ b/android/README.md @@ -0,0 +1,5 @@ +# Introduction + +Please refer to +https://k2-fsa.github.io/sherpa/onnx/android/index.html +for usage. diff --git a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt index 010be1f23..d34266957 100644 --- a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt +++ b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt @@ -1,3 +1,4 @@ +// Copyright (c) 2023 Xiaomi Corporation package com.k2fsa.sherpa.onnx import android.content.res.AssetManager diff --git a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt index 3060450d6..dca399840 100644 --- a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt +++ b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt @@ -1,3 +1,4 @@ +// Copyright (c) 2023 Xiaomi Corporation package com.k2fsa.sherpa.onnx import android.content.res.AssetManager diff --git a/android/SherpaOnnxVad/.gitignore b/android/SherpaOnnxVad/.gitignore new file mode 100644 index 000000000..aa724b770 --- /dev/null +++ b/android/SherpaOnnxVad/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/android/SherpaOnnxVad/.idea/.gitignore b/android/SherpaOnnxVad/.idea/.gitignore new file mode 100644 index 000000000..26d33521a --- /dev/null +++ b/android/SherpaOnnxVad/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/android/SherpaOnnxVad/.idea/compiler.xml b/android/SherpaOnnxVad/.idea/compiler.xml new file mode 100644 index 000000000..fb7f4a8a4 --- /dev/null +++ b/android/SherpaOnnxVad/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/.idea/gradle.xml b/android/SherpaOnnxVad/.idea/gradle.xml new file mode 100644 index 000000000..a2d7c2133 --- /dev/null +++ b/android/SherpaOnnxVad/.idea/gradle.xml @@ -0,0 +1,19 @@ + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/.idea/misc.xml b/android/SherpaOnnxVad/.idea/misc.xml new file mode 100644 index 000000000..bdd92780c --- /dev/null +++ b/android/SherpaOnnxVad/.idea/misc.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/.idea/vcs.xml b/android/SherpaOnnxVad/.idea/vcs.xml new file mode 100644 index 000000000..b2bdec2d7 --- /dev/null +++ b/android/SherpaOnnxVad/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/.gitignore b/android/SherpaOnnxVad/app/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/android/SherpaOnnxVad/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/build.gradle b/android/SherpaOnnxVad/app/build.gradle new file mode 100644 index 000000000..c9b51a6fd --- /dev/null +++ b/android/SherpaOnnxVad/app/build.gradle @@ -0,0 +1,44 @@ +plugins { + id 'com.android.application' + id 'org.jetbrains.kotlin.android' +} + +android { + namespace 'com.k2fsa.sherpa.onnx' + compileSdk 33 + + defaultConfig { + applicationId "com.k2fsa.sherpa.onnx" + minSdk 21 + targetSdk 33 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = '1.8' + } +} + +dependencies { + + implementation 'androidx.core:core-ktx:1.7.0' + implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'com.google.android.material:material:1.9.0' + implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test.ext:junit:1.1.5' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' +} \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/proguard-rules.pro b/android/SherpaOnnxVad/app/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/android/SherpaOnnxVad/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt b/android/SherpaOnnxVad/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt new file mode 100644 index 000000000..183383202 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.k2fsa.sherpa.onnx + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.k2fsa.sherpa.onnx", appContext.packageName) + } +} \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/src/main/AndroidManifest.xml b/android/SherpaOnnxVad/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..4c591cc53 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/AndroidManifest.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/src/main/assets/.gitignore b/android/SherpaOnnxVad/app/src/main/assets/.gitignore new file mode 100644 index 000000000..e1a699ac3 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/assets/.gitignore @@ -0,0 +1 @@ +*.onnx diff --git a/android/SherpaOnnxVad/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnxVad/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt new file mode 100644 index 000000000..eb847b020 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -0,0 +1,174 @@ +package com.k2fsa.sherpa.onnx + +import android.Manifest +import android.content.pm.PackageManager +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import android.os.Bundle +import android.util.Log +import android.view.View +import android.widget.Button +import androidx.appcompat.app.AppCompatActivity +import androidx.core.app.ActivityCompat +import kotlin.concurrent.thread + + +private const val TAG = "sherpa-onnx" +private const val REQUEST_RECORD_AUDIO_PERMISSION = 200 + +class MainActivity : AppCompatActivity() { + + private lateinit var recordButton: Button + private lateinit var circle: View + + private lateinit var vad: Vad + + private var audioRecord: AudioRecord? = null + private var recordingThread: Thread? = null + private val audioSource = MediaRecorder.AudioSource.MIC + private val sampleRateInHz = 16000 + private val channelConfig = AudioFormat.CHANNEL_IN_MONO + + // Note: We don't use AudioFormat.ENCODING_PCM_FLOAT + // since the AudioRecord.read(float[]) needs API level >= 23 + // but we are targeting API level >= 21 + private val audioFormat = AudioFormat.ENCODING_PCM_16BIT + + private val permissions: Array = arrayOf(Manifest.permission.RECORD_AUDIO) + + @Volatile + private var isRecording: Boolean = false + + override fun onRequestPermissionsResult( + requestCode: Int, permissions: Array, grantResults: IntArray + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + val permissionToRecordAccepted = if (requestCode == REQUEST_RECORD_AUDIO_PERMISSION) { + grantResults[0] == PackageManager.PERMISSION_GRANTED + } else { + false + } + + if (!permissionToRecordAccepted) { + Log.e(TAG, "Audio record is disallowed") + finish() + } + + Log.i(TAG, "Audio record is permitted") + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + + Log.i(TAG, "Start to initialize model") + initVadModel() + Log.i(TAG, "Finished initializing model") + + circle= findViewById(R.id.powerCircle) + + recordButton = findViewById(R.id.record_button) + recordButton.setOnClickListener { onclick() } + } + + private fun onclick() { + if (!isRecording) { + val ret = initMicrophone() + if (!ret) { + Log.e(TAG, "Failed to initialize microphone") + return + } + Log.i(TAG, "state: ${audioRecord?.state}") + audioRecord!!.startRecording() + recordButton.setText(R.string.stop) + isRecording = true + + vad.reset() + recordingThread = thread(true) { + processSamples() + } + Log.i(TAG, "Started recording") + onVad(false) + + } else { + isRecording = false + + audioRecord!!.stop() + audioRecord!!.release() + audioRecord = null + + recordButton.setText(R.string.start) + onVad(false) + Log.i(TAG, "Stopped recording") + } + } + + private fun onVad(isSpeech: Boolean) { + if(isSpeech) { + circle.background = resources.getDrawable(R.drawable.red_circle) + } else { + circle.background = resources.getDrawable(R.drawable.black_circle) + } + } + + private fun initVadModel() { + val type = 0 + println("Select VAD model type ${type}") + val config = getVadModelConfig(type) + + vad = Vad( + assetManager = application.assets, + config = config!!, + ) + } + + private fun initMicrophone(): Boolean { + if (ActivityCompat.checkSelfPermission( + this, Manifest.permission.RECORD_AUDIO + ) != PackageManager.PERMISSION_GRANTED + ) { + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + return false + } + + val numBytes = AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat) + Log.i( + TAG, "buffer size in milliseconds: ${numBytes * 1000.0f / sampleRateInHz}" + ) + + audioRecord = AudioRecord( + audioSource, + sampleRateInHz, + channelConfig, + audioFormat, + numBytes * 2 // a sample has two bytes as we are using 16-bit PCM + ) + return true + } + + private fun processSamples() { + Log.i(TAG, "processing samples") + + val bufferSize = 512 // in samples + val buffer = ShortArray(bufferSize) + + while (isRecording) { + val ret = audioRecord?.read(buffer, 0, buffer.size) + if (ret != null && ret > 0) { + val samples = FloatArray(ret) { buffer[it] / 32768.0f } + + vad.acceptWaveform(samples) + while(!vad.empty()) {vad.pop();} + + val isSpeechDetected = vad.isSpeechDetected() + + runOnUiThread { + onVad(isSpeechDetected) + } + } + } + } +} \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/src/main/java/com/k2fsa/sherpa/onnx/Vad.kt b/android/SherpaOnnxVad/app/src/main/java/com/k2fsa/sherpa/onnx/Vad.kt new file mode 100644 index 000000000..fc21593ab --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/java/com/k2fsa/sherpa/onnx/Vad.kt @@ -0,0 +1,101 @@ +// Copyright (c) 2023 Xiaomi Corporation +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class SileroVadModelConfig( + var model: String, + var threshold: Float = 0.5F, + var minSilenceDuration: Float = 0.25F, + var minSpeechDuration: Float = 0.25F, + var windowSize: Int = 512, +) + +data class VadModelConfig( + var sileroVadModelConfig: SileroVadModelConfig, + var sampleRate: Int = 16000, + var numThreads: Int = 1, + var provider: String = "cpu", + var debug: Boolean = false, +) + +class Vad( + assetManager: AssetManager? = null, + var config: VadModelConfig, +) { + private val ptr: Long + + init { + if (assetManager != null) { + ptr = new(assetManager, config) + } else { + ptr = newFromFile(config) + } + } + + protected fun finalize() { + delete(ptr) + } + + fun acceptWaveform(samples: FloatArray) = acceptWaveform(ptr, samples) + + fun empty(): Boolean = empty(ptr) + fun pop() = pop(ptr) + + // return an array containing + // [start: Int, samples: FloatArray] + fun front() = front(ptr) + + fun isSpeechDetected(): Boolean = isSpeechDetected(ptr) + + fun reset() = reset(ptr) + + private external fun delete(ptr: Long) + + private external fun new( + assetManager: AssetManager, + config: VadModelConfig, + ): Long + + private external fun newFromFile( + config: VadModelConfig, + ): Long + + private external fun acceptWaveform(ptr: Long, samples: FloatArray) + private external fun empty(ptr: Long): Boolean + private external fun pop(ptr: Long) + private external fun front(ptr: Long): Array + private external fun isSpeechDetected(ptr: Long): Boolean + private external fun reset(ptr: Long) + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} + +// Please visit +// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx +// to download silero_vad.onnx +// and put it inside the assets/ +// directory +fun getVadModelConfig(type: Int): VadModelConfig? { + when (type) { + 0 -> { + return VadModelConfig( + sileroVadModelConfig = SileroVadModelConfig( + model = "silero_vad.onnx", + threshold = 0.5F, + minSilenceDuration = 0.25F, + minSpeechDuration = 0.25F, + windowSize = 512, + ), + sampleRate = 16000, + numThreads = 1, + provider = "cpu", + ) + } + } + return null; +} diff --git a/android/SherpaOnnxVad/app/src/main/jniLibs/.gitignore b/android/SherpaOnnxVad/app/src/main/jniLibs/.gitignore new file mode 100644 index 000000000..140f8cf80 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/jniLibs/.gitignore @@ -0,0 +1 @@ +*.so diff --git a/android/SherpaOnnxVad/app/src/main/jniLibs/arm64-v8a/.gitignore b/android/SherpaOnnxVad/app/src/main/jniLibs/arm64-v8a/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxVad/app/src/main/jniLibs/armeabi-v7a/.gitignore b/android/SherpaOnnxVad/app/src/main/jniLibs/armeabi-v7a/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxVad/app/src/main/jniLibs/x86/.gitignore b/android/SherpaOnnxVad/app/src/main/jniLibs/x86/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxVad/app/src/main/jniLibs/x86_64/.gitignore b/android/SherpaOnnxVad/app/src/main/jniLibs/x86_64/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxVad/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/android/SherpaOnnxVad/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..2b068d114 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/src/main/res/drawable/black_circle.xml b/android/SherpaOnnxVad/app/src/main/res/drawable/black_circle.xml new file mode 100644 index 000000000..c20dcdd83 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/res/drawable/black_circle.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/src/main/res/drawable/ic_launcher_background.xml b/android/SherpaOnnxVad/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..07d5da9cb --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/SherpaOnnxVad/app/src/main/res/drawable/red_circle.xml b/android/SherpaOnnxVad/app/src/main/res/drawable/red_circle.xml new file mode 100644 index 000000000..e7fc58ba4 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/res/drawable/red_circle.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxVad/app/src/main/res/layout/activity_main.xml b/android/SherpaOnnxVad/app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..cb8294da1 --- /dev/null +++ b/android/SherpaOnnxVad/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,43 @@ + + + + + + + + + + +