Skip to content

Commit

Permalink
Merge pull request #166 from solaoi/feature_add-actions
Browse files Browse the repository at this point in the history
Feature add actions
  • Loading branch information
solaoi authored Sep 10, 2024
2 parents 618a2d2 + df4b389 commit f100997
Show file tree
Hide file tree
Showing 27 changed files with 699 additions and 394 deletions.
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "lycoris",
"private": true,
"version": "0.9.19",
"version": "0.9.20",
"type": "module",
"license": "MIT",
"engines": {
Expand All @@ -18,15 +18,15 @@
"dependencies": {
"@tauri-apps/api": "^1.6.0",
"dayjs": "^1.11.13",
"markdown-to-jsx": "^7.5.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-h5-audio-player": "^3.9.3",
"react-medium-image-zoom": "^5.2.8",
"react-toastify": "^10.0.5",
"recoil": "^0.7.7",
"tauri-plugin-sql-api": "github:tauri-apps/tauri-plugin-sql#release",
"zenn-content-css": "^0.1.155",
"zenn-markdown-html": "^0.1.155"
"zenn-content-css": "^0.1.155"
},
"devDependencies": {
"@tauri-apps/cli": "^1.6.1",
Expand Down
2 changes: 1 addition & 1 deletion src-tauri/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion src-tauri/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "lycoris"
version = "0.9.19"
version = "0.9.20"
description = "Lycoris is an offline voice memo"
authors = ["solaoi"]
license = "MIT"
Expand Down
3 changes: 2 additions & 1 deletion src-tauri/migrations/001.sql
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ CREATE TABLE notes (
CREATE TABLE speeches (
id INTEGER PRIMARY KEY AUTOINCREMENT,
speech_type TEXT,
-- speech|memo|screenshot
-- speech|memo|screenshot|action
created_at_unixtime INTEGER DEFAULT (CAST(strftime('%s', 'now') AS INTEGER)),
content TEXT,
content_2 TEXT,
wav TEXT,
model TEXT,
-- manual|vosk|whisper
Expand Down
17 changes: 17 additions & 0 deletions src-tauri/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use urlencoding::decode;

mod module;
use module::{
action,
chat_online::ChatOnline,
deleter::NoteDeleter,
device::{self, Device},
Expand Down Expand Up @@ -122,6 +123,21 @@ fn has_microphone_permission_command(window: Window) -> bool {
permissions::has_microphone_permission(window)
}

#[tauri::command]
fn execute_action_command(window: Window, note_id: u64) {
std::thread::spawn(move || {
if action::initialize_action(window.app_handle().clone(), note_id) {
let mut lock = action::SINGLETON_INSTANCE.lock().unwrap();
if let Some(singleton) = lock.as_mut() {
singleton.execute();
}
} else {
println!("Action is already initialized and executing. Skipping.");
}
action::drop_action();
});
}

#[tauri::command]
fn start_command(
state: State<'_, RecordState>,
Expand Down Expand Up @@ -343,6 +359,7 @@ fn main() {
has_accessibility_permission_command,
has_screen_capture_permission_command,
has_microphone_permission_command,
execute_action_command,
start_command,
stop_command,
start_trace_command,
Expand Down
200 changes: 200 additions & 0 deletions src-tauri/src/module/action.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
use std::sync::Mutex;

use reqwest::{
header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE},
Client,
};
use serde_json::{json, Value};
use tauri::{AppHandle, Manager};

use super::sqlite::{Content, Sqlite};

pub struct Action {
app_handle: AppHandle,
sqlite: Sqlite,
note_id: u64,
model: String,
token: String,
}

impl Action {
pub fn new(app_handle: AppHandle, note_id: u64) -> Self {
let sqlite = Sqlite::new();
let token = sqlite.select_whisper_token().unwrap();
let model = sqlite
.select_ai_model()
.unwrap_or_else(|_| "gpt-4o-mini".to_string());
Self {
app_handle,
sqlite,
note_id,
model,
token,
}
}

#[tokio::main]
async fn request_gpt(
model: String,
question: String,
contents: Vec<Content>,
token: String,
) -> Result<String, Box<dyn std::error::Error>> {
let url = "https://api.openai.com/v1/chat/completions";
let temperature = 0;

let client = Client::new();

let mut headers = HeaderMap::new();
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {}", token))?,
);
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));

let mut messages: Vec<Value> = Vec::new();
for content in contents.iter() {
if content.speech_type == "action" {
messages.push(json!({
"role": "user",
"content": content.content.clone()
}));
messages.push(json!({
"role": "assistant",
"content": content.content_2.clone()
}));
} else {
messages.push(json!({
"role": "user",
"content": content.content.clone()
}));
}
}
messages.push(json!({
"role": "user",
"content": question
}));

// for debugging
// println!("messages: {:?}", messages);

let post_body = json!({
"model": model,
"temperature": temperature,
"messages": messages
});

let response = client
.post(url)
.headers(headers)
.json(&post_body)
.send()
.await?;

let status = response.status();
let json_response: Value = response.json().await?;

let response_text = if status == 200 {
json_response["choices"][0]["message"]["content"]
.as_str()
.unwrap_or("choices[0].message.content field not found")
.to_string()
} else {
json_response.to_string()
};

Ok(response_text)
}

pub fn execute(&mut self) {
if self.token == "" {
println!("whisper token is empty, so skipping...");
return;
}
let mut is_executing = IS_EXECUTING.lock().unwrap();
*is_executing = true;

while let Ok(action) = self.sqlite.select_first_unexecuted_action(self.note_id) {
match self
.sqlite
.select_has_no_permission_of_execute_action(self.note_id, action.id)
{
Ok(permissions) => {
if permissions.is_empty() || permissions.iter().any(|p| p.model == "whisper") {
match self.sqlite.select_contents_by(self.note_id, action.id) {
Ok(contents) => {
match Self::request_gpt(
self.model.clone(),
action.content,
contents,
self.token.clone(),
) {
Ok(answer) => {
match self
.sqlite
.update_action_content_2(action.id, answer.clone())
{
Ok(result) => {
let _ = self
.app_handle
.emit_all("actionExecuted", result);
}
Err(e) => {
println!(
"Error updating action content_2: {:?}",
e
);
break;
}
}
}
Err(_) => {
println!("gpt api is temporarily failed, so skipping...");
break;
}
}
}
Err(e) => {
println!("Error selecting contents: {:?}", e);
break;
}
}
} else {
println!("has_no_permission_of_execute_action is false, so skipping...");
break;
}
}
Err(e) => {
println!("Error checking permissions: {:?}", e);
break;
}
}
}

*is_executing = false;
}
}

pub static SINGLETON_INSTANCE: Mutex<Option<Action>> = Mutex::new(None);
pub static IS_EXECUTING: Mutex<bool> = Mutex::new(false);

pub fn initialize_action(app_handle: AppHandle, note_id: u64) -> bool {
let mut singleton = SINGLETON_INSTANCE.lock().unwrap();
let is_executing = IS_EXECUTING.lock().unwrap();
if singleton.is_none() {
*singleton = Some(Action::new(app_handle, note_id));
true
} else if *is_executing {
false
} else {
true
}
}

pub fn drop_action() {
let is_executing = IS_EXECUTING.lock().unwrap();
if !*is_executing {
let mut singleton = SINGLETON_INSTANCE.lock().unwrap();
*singleton = None;
}
}
3 changes: 2 additions & 1 deletion src-tauri/src/module/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,5 @@ pub mod transcription_online;
pub mod translation_ja;
pub mod translation_ja_high;
mod writer;
pub mod screenshot;
pub mod screenshot;
pub mod action;
81 changes: 81 additions & 0 deletions src-tauri/src/module/sqlite.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,24 @@ pub struct Updated {
pub content: String,
}

#[derive(Debug, Clone, serde::Serialize)]
pub struct UnexecutedAction {
pub id: u16,
pub content: String,
}

#[derive(Debug, Clone, serde::Serialize)]
pub struct Content {
pub speech_type: String,
pub content: String,
pub content_2: String,
}

#[derive(Debug, Clone, serde::Serialize)]
pub struct Permission {
pub model: String,
}

impl Sqlite {
pub fn new() -> Self {
let data_dir = data_dir().unwrap_or(PathBuf::from("./"));
Expand Down Expand Up @@ -172,6 +190,69 @@ impl Sqlite {
);
}

pub fn select_contents_by(
&self,
note_id: u64,
id: u16,
) -> Result<Vec<Content>, rusqlite::Error> {
let mut stmt = self.conn.prepare("SELECT speech_type,content,content_2 FROM speeches WHERE note_id = ?1 AND id < ?2 ORDER BY created_at_unixtime ASC").unwrap();
let results = stmt
.query_map(params![note_id, id], |row| {
Ok(Content {
speech_type: row.get_unwrap(0),
content: row.get_unwrap(1),
content_2: row.get(2).unwrap_or_default(),
})
})
.unwrap()
.collect::<Result<Vec<_>, rusqlite::Error>>();
results
}

pub fn select_first_unexecuted_action(
&self,
note_id: u64,
) -> Result<UnexecutedAction, rusqlite::Error> {
return self.conn.query_row("SELECT id, content FROM speeches WHERE speech_type = \"action\" AND content_2 IS NULL AND note_id = ?1 ORDER BY created_at_unixtime ASC LIMIT 1",
params![note_id],
|row| Ok(UnexecutedAction{id: row.get_unwrap(0), content: row.get_unwrap(1)}),
);
}

pub fn select_has_no_permission_of_execute_action(
&self,
note_id: u64,
id: u16,
) -> Result<Vec<Permission>, rusqlite::Error> {
let mut stmt = self.conn.prepare("SELECT model FROM speeches WHERE id = (SELECT MAX(id) FROM speeches WHERE note_id = ?1 AND id < ?2 AND model != \"manual\") OR id = (SELECT MIN(id) FROM speeches WHERE note_id = ?1 AND id > ?2 AND model = \"whisper\")").unwrap();
let results = stmt
.query_map(params![note_id, id], |row| {
Ok(Permission {
model: row.get_unwrap(0),
})
})
.unwrap()
.collect::<Result<Vec<_>, rusqlite::Error>>();
results
}

pub fn update_action_content_2(
&self,
id: u16,
content_2: String,
) -> Result<Updated, rusqlite::Error> {
match self.conn.execute(
"UPDATE speeches SET content_2 = ?1 WHERE id = ?2",
params![content_2, id],
) {
Ok(_) => Ok(Updated {
id,
content: content_2,
}),
Err(err) => Err(err),
}
}

pub fn update_has_accessed_screen_capture_permission(&self) -> Result<usize, rusqlite::Error> {
self.conn.execute(
"UPDATE settings SET setting_status = \"has_accessed\" WHERE setting_name = \"settingHasAccessedScreenCapturePermission\"",
Expand Down
Loading

0 comments on commit f100997

Please sign in to comment.