chore: initial commit
This commit is contained in:
77
.gitignore
vendored
Normal file
77
.gitignore
vendored
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
# SQLite databases
|
||||||
|
*.db
|
||||||
|
*.db-shm
|
||||||
|
*.db-wal
|
||||||
|
*.sqlite
|
||||||
|
*.sqlite3
|
||||||
|
|
||||||
|
# Compressed files
|
||||||
|
*.zip
|
||||||
|
*.tar.gz
|
||||||
|
*.7z
|
||||||
|
*.rar
|
||||||
|
|
||||||
|
# Rust/Cargo
|
||||||
|
target/
|
||||||
|
Cargo.lock # Remove this line if this is a binary crate, keep for libraries
|
||||||
|
**/*.rs.bk
|
||||||
|
*.pdb
|
||||||
|
|
||||||
|
# macOS
|
||||||
|
.DS_Store
|
||||||
|
.AppleDouble
|
||||||
|
.LSOverride
|
||||||
|
._*
|
||||||
|
|
||||||
|
# Windows
|
||||||
|
Thumbs.db
|
||||||
|
Thumbs.db:encryptable
|
||||||
|
ehthumbs.db
|
||||||
|
Desktop.ini
|
||||||
|
|
||||||
|
# Linux
|
||||||
|
*~
|
||||||
|
.directory
|
||||||
|
|
||||||
|
# IDEs and editors
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
.project
|
||||||
|
.classpath
|
||||||
|
.settings/
|
||||||
|
*.sublime-project
|
||||||
|
*.sublime-workspace
|
||||||
|
|
||||||
|
# Environment and config files that may contain secrets
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
.env.*.local
|
||||||
|
config.toml # Remove this if you want to track config
|
||||||
|
*.local.toml
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
*.log
|
||||||
|
logs/
|
||||||
|
|
||||||
|
# Temporary files
|
||||||
|
tmp/
|
||||||
|
temp/
|
||||||
|
*.tmp
|
||||||
|
|
||||||
|
# Generated documentation
|
||||||
|
docs/book/
|
||||||
|
target/doc/
|
||||||
|
|
||||||
|
# OS-specific network storage
|
||||||
|
.Trashes
|
||||||
|
.VolumeIcon.icns
|
||||||
|
.com.apple.timemachine.donotpresent
|
||||||
|
.fseventsd
|
||||||
|
.Spotlight-V100
|
||||||
|
.TemporaryItems
|
||||||
|
|
||||||
|
# Project-specific (based on your untracked files)
|
||||||
|
emotion-gradient-config-*.json
|
||||||
8506
Cargo.lock
generated
Normal file
8506
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
51
Cargo.toml
Normal file
51
Cargo.toml
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
[workspace]
|
||||||
|
members = ["crates/lib", "crates/server", "crates/client", "crates/sync-macros"]
|
||||||
|
resolver = "2"
|
||||||
|
|
||||||
|
[workspace.package]
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[workspace.dependencies]
|
||||||
|
# Async runtime
|
||||||
|
tokio = { version = "1", features = ["full"] }
|
||||||
|
tokio-stream = "0.1"
|
||||||
|
|
||||||
|
# Iroh - P2P networking and gossip
|
||||||
|
iroh = { version = "0.95.0",features = ["discovery-local-network"] }
|
||||||
|
iroh-gossip = "0.95.0"
|
||||||
|
|
||||||
|
# Database
|
||||||
|
rusqlite = "0.37.0"
|
||||||
|
|
||||||
|
# Serialization
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0"
|
||||||
|
toml = "0.9"
|
||||||
|
|
||||||
|
# Error handling
|
||||||
|
thiserror = "2.0"
|
||||||
|
anyhow = "1.0"
|
||||||
|
|
||||||
|
# Date/time
|
||||||
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
tracing = "0.1"
|
||||||
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|
||||||
|
# Random
|
||||||
|
rand = "0.8"
|
||||||
|
|
||||||
|
# ML/AI
|
||||||
|
candle-core = "0.8"
|
||||||
|
candle-nn = "0.8"
|
||||||
|
candle-transformers = "0.8"
|
||||||
|
tokenizers = "0.20"
|
||||||
|
hf-hub = "0.3"
|
||||||
|
|
||||||
|
# Bevy
|
||||||
|
bevy = "0.17"
|
||||||
|
|
||||||
|
# Synchronization
|
||||||
|
parking_lot = "0.12"
|
||||||
|
crdts = "7.3"
|
||||||
18
config.toml
Normal file
18
config.toml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
[database]
|
||||||
|
path = "./us.db"
|
||||||
|
chat_db_path = "./crates/lib/chat.db"
|
||||||
|
|
||||||
|
[services]
|
||||||
|
poll_interval_ms = 1000
|
||||||
|
training_set_sample_rate = 0.05
|
||||||
|
|
||||||
|
[models]
|
||||||
|
embedding_model = "Qwen/Qwen3-Embedding-0.6B"
|
||||||
|
emotion_model = "SamLowe/roberta-base-go_emotions"
|
||||||
|
|
||||||
|
[tailscale]
|
||||||
|
hostname = "lonni-daemon"
|
||||||
|
state_dir = "./tailscale-state"
|
||||||
|
|
||||||
|
[grpc]
|
||||||
|
port = 50051
|
||||||
1
crates/client/.gitignore
vendored
Normal file
1
crates/client/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
/target
|
||||||
43
crates/client/Cargo.toml
Normal file
43
crates/client/Cargo.toml
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
[package]
|
||||||
|
name = "client"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "client"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
# Bevy
|
||||||
|
bevy = { version = "0.17", default-features = false, features = [
|
||||||
|
"bevy_winit",
|
||||||
|
"bevy_render",
|
||||||
|
"bevy_core_pipeline",
|
||||||
|
"bevy_sprite",
|
||||||
|
"bevy_ui",
|
||||||
|
"bevy_text",
|
||||||
|
"png",
|
||||||
|
"x11",
|
||||||
|
] }
|
||||||
|
|
||||||
|
# Iroh - P2P networking and gossip
|
||||||
|
iroh = { workspace = true }
|
||||||
|
iroh-gossip = { workspace = true }
|
||||||
|
|
||||||
|
# Async runtime
|
||||||
|
tokio = { version = "1", features = ["full"] }
|
||||||
|
|
||||||
|
# Serialization
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0"
|
||||||
|
|
||||||
|
# Error handling
|
||||||
|
thiserror = "2.0"
|
||||||
|
anyhow = "1.0"
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
tracing = "0.1"
|
||||||
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|
||||||
|
# Local dependencies
|
||||||
|
lib = { path = "../lib" }
|
||||||
14
crates/client/src/lib.rs
Normal file
14
crates/client/src/lib.rs
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
pub fn add(left: u64, right: u64) -> u64 {
|
||||||
|
left + right
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn it_works() {
|
||||||
|
let result = add(2, 2);
|
||||||
|
assert_eq!(result, 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
24
crates/client/src/main.rs
Normal file
24
crates/client/src/main.rs
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
use bevy::prelude::*;
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||||
|
.init();
|
||||||
|
|
||||||
|
// Start Bevy app
|
||||||
|
App::new()
|
||||||
|
.add_plugins(DefaultPlugins)
|
||||||
|
.add_systems(Startup, setup)
|
||||||
|
.add_systems(Update, sync_system)
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn setup(mut commands: Commands) {
|
||||||
|
commands.spawn(Camera2d);
|
||||||
|
info!("Client started");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sync_system() {
|
||||||
|
// TODO: Implement gossip sync for client
|
||||||
|
}
|
||||||
4
crates/lib/.gitignore
vendored
Normal file
4
crates/lib/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
/target
|
||||||
|
chat.db
|
||||||
|
*.db-shm
|
||||||
|
*.db-wal
|
||||||
20
crates/lib/Cargo.toml
Normal file
20
crates/lib/Cargo.toml
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
[package]
|
||||||
|
name = "lib"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
rusqlite = { version = "0.37.0", features = ["bundled"] }
|
||||||
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
|
thiserror = "2.0"
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json.workspace = true
|
||||||
|
crdts.workspace = true
|
||||||
|
anyhow.workspace = true
|
||||||
|
sync-macros = { path = "../sync-macros" }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tokio.workspace = true
|
||||||
|
iroh.workspace = true
|
||||||
|
iroh-gossip.workspace = true
|
||||||
|
futures-lite = "2.0"
|
||||||
139
crates/lib/src/db.rs
Normal file
139
crates/lib/src/db.rs
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
use crate::error::Result;
|
||||||
|
use crate::models::*;
|
||||||
|
use rusqlite::{Connection, OpenFlags, Row, params};
|
||||||
|
|
||||||
|
pub struct ChatDb {
|
||||||
|
conn: Connection,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatDb {
|
||||||
|
/// Open a connection to the chat database in read-only mode
|
||||||
|
pub fn open(path: &str) -> Result<Self> {
|
||||||
|
let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||||
|
Ok(Self { conn })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get messages from the conversation with +31 6 39 13 29 13
|
||||||
|
///
|
||||||
|
/// Returns messages from January 1, 2024 to present from the conversation
|
||||||
|
/// with the specified Dutch phone number.
|
||||||
|
///
|
||||||
|
/// # Arguments
|
||||||
|
///
|
||||||
|
/// * `start_date` - Start date (defaults to January 1, 2024 if None)
|
||||||
|
/// * `end_date` - End date (defaults to current time if None)
|
||||||
|
pub fn get_our_messages(
|
||||||
|
&self,
|
||||||
|
start_date: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
|
end_date: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
|
) -> Result<Vec<Message>> {
|
||||||
|
use chrono::{TimeZone, Utc};
|
||||||
|
|
||||||
|
// Default date range: January 1, 2024 to now
|
||||||
|
let start =
|
||||||
|
start_date.unwrap_or_else(|| Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap());
|
||||||
|
let end = end_date.unwrap_or_else(|| Utc::now());
|
||||||
|
|
||||||
|
// Convert to Apple timestamps (nanoseconds since 2001-01-01)
|
||||||
|
let start_timestamp = datetime_to_apple_timestamp(start);
|
||||||
|
let end_timestamp = datetime_to_apple_timestamp(end);
|
||||||
|
|
||||||
|
// The phone number might be stored with or without spaces
|
||||||
|
let phone_with_spaces = "+31 6 39 13 29 13";
|
||||||
|
let phone_without_spaces = "+31639132913";
|
||||||
|
|
||||||
|
// Find the chat with this phone number (try both formats)
|
||||||
|
let chat = self
|
||||||
|
.get_chat_for_phone_number(phone_with_spaces)
|
||||||
|
.or_else(|_| self.get_chat_for_phone_number(phone_without_spaces))?;
|
||||||
|
|
||||||
|
// Get messages from this chat within the date range
|
||||||
|
let mut stmt = self.conn.prepare(
|
||||||
|
"SELECT m.ROWID, m.guid, m.text, m.service, m.handle_id, m.date, m.date_read, m.date_delivered,
|
||||||
|
m.is_from_me, m.is_read, m.is_delivered, m.is_sent, m.is_emote, m.is_audio_message,
|
||||||
|
m.cache_has_attachments, m.associated_message_guid, m.associated_message_type,
|
||||||
|
m.thread_originator_guid, m.reply_to_guid, m.is_spam
|
||||||
|
FROM message m
|
||||||
|
INNER JOIN chat_message_join cmj ON m.ROWID = cmj.message_id
|
||||||
|
WHERE cmj.chat_id = ?
|
||||||
|
AND m.date >= ?
|
||||||
|
AND m.date <= ?
|
||||||
|
ORDER BY m.date ASC"
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let messages = stmt
|
||||||
|
.query_map(
|
||||||
|
params![chat.rowid, start_timestamp, end_timestamp],
|
||||||
|
map_message_row,
|
||||||
|
)?
|
||||||
|
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||||
|
|
||||||
|
Ok(messages)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper function to find the largest chat with a specific phone number
|
||||||
|
fn get_chat_for_phone_number(&self, phone_number: &str) -> Result<Chat> {
|
||||||
|
let mut stmt = self.conn.prepare(
|
||||||
|
"SELECT c.ROWID, c.guid, c.chat_identifier, c.service_name, c.display_name,
|
||||||
|
c.group_id, c.room_name, c.is_archived, c.is_filtered,
|
||||||
|
c.last_read_message_timestamp, COUNT(cmj.message_id) as msg_count
|
||||||
|
FROM chat c
|
||||||
|
INNER JOIN chat_handle_join chj ON c.ROWID = chj.chat_id
|
||||||
|
INNER JOIN handle h ON chj.handle_id = h.ROWID
|
||||||
|
INNER JOIN chat_message_join cmj ON c.ROWID = cmj.chat_id
|
||||||
|
WHERE h.id = ?
|
||||||
|
GROUP BY c.ROWID
|
||||||
|
ORDER BY msg_count DESC
|
||||||
|
LIMIT 1"
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let chat = stmt.query_row(params![phone_number], |row| {
|
||||||
|
Ok(Chat {
|
||||||
|
rowid: row.get(0)?,
|
||||||
|
guid: row.get(1)?,
|
||||||
|
chat_identifier: row.get(2)?,
|
||||||
|
service_name: row.get(3)?,
|
||||||
|
display_name: row.get(4)?,
|
||||||
|
group_id: row.get(5)?,
|
||||||
|
room_name: row.get(6)?,
|
||||||
|
is_archived: row.get::<_, i64>(7)? != 0,
|
||||||
|
is_filtered: row.get::<_, i64>(8)? != 0,
|
||||||
|
last_read_message_timestamp: row.get::<_, Option<i64>>(9)?.map(apple_timestamp_to_datetime),
|
||||||
|
})
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(chat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to map database rows to structs
|
||||||
|
fn map_message_row(row: &Row) -> rusqlite::Result<Message> {
|
||||||
|
Ok(Message {
|
||||||
|
rowid: row.get(0)?,
|
||||||
|
guid: row.get(1)?,
|
||||||
|
text: row.get(2)?,
|
||||||
|
service: row.get(3)?,
|
||||||
|
handle_id: row.get(4)?,
|
||||||
|
date: row
|
||||||
|
.get::<_, Option<i64>>(5)?
|
||||||
|
.map(apple_timestamp_to_datetime),
|
||||||
|
date_read: row
|
||||||
|
.get::<_, Option<i64>>(6)?
|
||||||
|
.map(apple_timestamp_to_datetime),
|
||||||
|
date_delivered: row
|
||||||
|
.get::<_, Option<i64>>(7)?
|
||||||
|
.map(apple_timestamp_to_datetime),
|
||||||
|
is_from_me: row.get::<_, i64>(8)? != 0,
|
||||||
|
is_read: row.get::<_, i64>(9)? != 0,
|
||||||
|
is_delivered: row.get::<_, i64>(10)? != 0,
|
||||||
|
is_sent: row.get::<_, i64>(11)? != 0,
|
||||||
|
is_emote: row.get::<_, i64>(12)? != 0,
|
||||||
|
is_audio_message: row.get::<_, i64>(13)? != 0,
|
||||||
|
cache_has_attachments: row.get::<_, i64>(14)? != 0,
|
||||||
|
associated_message_guid: row.get(15)?,
|
||||||
|
associated_message_type: row.get(16)?,
|
||||||
|
thread_originator_guid: row.get(17)?,
|
||||||
|
reply_to_guid: row.get(18)?,
|
||||||
|
is_spam: row.get::<_, i64>(19)? != 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
15
crates/lib/src/error.rs
Normal file
15
crates/lib/src/error.rs
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
use thiserror::Error;
|
||||||
|
|
||||||
|
#[derive(Error, Debug)]
|
||||||
|
pub enum ChatDbError {
|
||||||
|
#[error("Database error: {0}")]
|
||||||
|
Database(#[from] rusqlite::Error),
|
||||||
|
|
||||||
|
#[error("Not found: {0}")]
|
||||||
|
NotFound(String),
|
||||||
|
|
||||||
|
#[error("Invalid data: {0}")]
|
||||||
|
InvalidData(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub type Result<T> = std::result::Result<T, ChatDbError>;
|
||||||
30
crates/lib/src/lib.rs
Normal file
30
crates/lib/src/lib.rs
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
//! Data access layer for iMessage chat.db
|
||||||
|
//!
|
||||||
|
//! This library provides a read-only interface to query messages from a specific conversation.
|
||||||
|
//!
|
||||||
|
//! # Safety
|
||||||
|
//!
|
||||||
|
//! All database connections are opened in read-only mode to prevent any
|
||||||
|
//! accidental modifications to your iMessage database.
|
||||||
|
//!
|
||||||
|
//! # Example
|
||||||
|
//!
|
||||||
|
//! ```no_run
|
||||||
|
//! use lib::ChatDb;
|
||||||
|
//!
|
||||||
|
//! let db = ChatDb::open("chat.db")?;
|
||||||
|
//!
|
||||||
|
//! // Get all messages from January 2024 to now
|
||||||
|
//! let messages = db.get_our_messages(None, None)?;
|
||||||
|
//! println!("Found {} messages", messages.len());
|
||||||
|
//! # Ok::<(), lib::ChatDbError>(())
|
||||||
|
//! ```
|
||||||
|
|
||||||
|
mod error;
|
||||||
|
mod models;
|
||||||
|
mod db;
|
||||||
|
pub mod sync;
|
||||||
|
|
||||||
|
pub use error::{ChatDbError, Result};
|
||||||
|
pub use models::{Message, Chat};
|
||||||
|
pub use db::ChatDb;
|
||||||
112
crates/lib/src/models.rs
Normal file
112
crates/lib/src/models.rs
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Represents a message in the iMessage database
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct Message {
|
||||||
|
pub rowid: i64,
|
||||||
|
pub guid: String,
|
||||||
|
pub text: Option<String>,
|
||||||
|
pub service: Option<String>,
|
||||||
|
pub handle_id: i64,
|
||||||
|
pub date: Option<DateTime<Utc>>,
|
||||||
|
pub date_read: Option<DateTime<Utc>>,
|
||||||
|
pub date_delivered: Option<DateTime<Utc>>,
|
||||||
|
pub is_from_me: bool,
|
||||||
|
pub is_read: bool,
|
||||||
|
pub is_delivered: bool,
|
||||||
|
pub is_sent: bool,
|
||||||
|
pub is_emote: bool,
|
||||||
|
pub is_audio_message: bool,
|
||||||
|
pub cache_has_attachments: bool,
|
||||||
|
pub associated_message_guid: Option<String>,
|
||||||
|
pub associated_message_type: i64,
|
||||||
|
pub thread_originator_guid: Option<String>,
|
||||||
|
pub reply_to_guid: Option<String>,
|
||||||
|
pub is_spam: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Represents a chat/conversation
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct Chat {
|
||||||
|
pub rowid: i64,
|
||||||
|
pub guid: String,
|
||||||
|
pub chat_identifier: Option<String>,
|
||||||
|
pub service_name: Option<String>,
|
||||||
|
pub display_name: Option<String>,
|
||||||
|
pub group_id: Option<String>,
|
||||||
|
pub room_name: Option<String>,
|
||||||
|
pub is_archived: bool,
|
||||||
|
pub is_filtered: bool,
|
||||||
|
pub last_read_message_timestamp: Option<DateTime<Utc>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper function to convert Apple's Cocoa timestamp (seconds since 2001-01-01) to DateTime
|
||||||
|
pub fn apple_timestamp_to_datetime(timestamp: i64) -> DateTime<Utc> {
|
||||||
|
// Apple's Cocoa timestamps are in nanoseconds since 2001-01-01 00:00:00 UTC
|
||||||
|
// Convert to Unix timestamp (seconds since 1970-01-01 00:00:00 UTC)
|
||||||
|
const APPLE_EPOCH_OFFSET: i64 = 978307200; // Seconds between 1970-01-01 and 2001-01-01
|
||||||
|
|
||||||
|
let seconds = timestamp / 1_000_000_000 + APPLE_EPOCH_OFFSET;
|
||||||
|
let nanos = (timestamp % 1_000_000_000) as u32;
|
||||||
|
|
||||||
|
DateTime::from_timestamp(seconds, nanos).unwrap_or_else(|| DateTime::from_timestamp(0, 0).unwrap())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper function to convert DateTime to Apple's Cocoa timestamp
|
||||||
|
pub fn datetime_to_apple_timestamp(dt: DateTime<Utc>) -> i64 {
|
||||||
|
const APPLE_EPOCH_OFFSET: i64 = 978307200;
|
||||||
|
|
||||||
|
let unix_timestamp = dt.timestamp();
|
||||||
|
let nanos = dt.timestamp_subsec_nanos() as i64;
|
||||||
|
|
||||||
|
(unix_timestamp - APPLE_EPOCH_OFFSET) * 1_000_000_000 + nanos
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use chrono::{Datelike, TimeZone, Timelike};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_apple_timestamp_to_datetime_zero() {
|
||||||
|
let dt = apple_timestamp_to_datetime(0);
|
||||||
|
assert_eq!(dt.year(), 2001);
|
||||||
|
assert_eq!(dt.month(), 1);
|
||||||
|
assert_eq!(dt.day(), 1);
|
||||||
|
assert_eq!(dt.hour(), 0);
|
||||||
|
assert_eq!(dt.minute(), 0);
|
||||||
|
assert_eq!(dt.second(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_apple_timestamp_to_datetime_known_value() {
|
||||||
|
let timestamp = 694224000000000000i64;
|
||||||
|
let dt = apple_timestamp_to_datetime(timestamp);
|
||||||
|
assert_eq!(dt.year(), 2023);
|
||||||
|
assert_eq!(dt.month(), 1);
|
||||||
|
assert_eq!(dt.day(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_apple_timestamp_roundtrip() {
|
||||||
|
let original = 694224000000000000i64;
|
||||||
|
let dt = apple_timestamp_to_datetime(original);
|
||||||
|
let converted_back = datetime_to_apple_timestamp(dt);
|
||||||
|
assert_eq!(original, converted_back);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_datetime_to_apple_timestamp_epoch() {
|
||||||
|
let dt = Utc.with_ymd_and_hms(2001, 1, 1, 0, 0, 0).unwrap();
|
||||||
|
let timestamp = datetime_to_apple_timestamp(dt);
|
||||||
|
assert_eq!(timestamp, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_negative_apple_timestamp() {
|
||||||
|
let timestamp = -31536000000000000i64;
|
||||||
|
let dt = apple_timestamp_to_datetime(timestamp);
|
||||||
|
assert_eq!(dt.year(), 2000);
|
||||||
|
}
|
||||||
|
}
|
||||||
165
crates/lib/src/sync.rs
Normal file
165
crates/lib/src/sync.rs
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::ops::{Deref, DerefMut};
|
||||||
|
|
||||||
|
// Re-export the macros
|
||||||
|
pub use sync_macros::{synced, Synced};
|
||||||
|
|
||||||
|
// Re-export common CRDT types from the crdts library
|
||||||
|
pub use crdts::{
|
||||||
|
ctx::ReadCtx,
|
||||||
|
lwwreg::LWWReg,
|
||||||
|
map::Map,
|
||||||
|
orswot::Orswot,
|
||||||
|
CmRDT, CvRDT,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub type NodeId = String;
|
||||||
|
|
||||||
|
/// Transparent wrapper for synced values
|
||||||
|
///
|
||||||
|
/// This wraps any value with LWW semantics but allows you to use it like a normal value
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SyncedValue<T: Clone> {
|
||||||
|
value: T,
|
||||||
|
timestamp: DateTime<Utc>,
|
||||||
|
node_id: NodeId,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Clone> SyncedValue<T> {
|
||||||
|
pub fn new(value: T, node_id: NodeId) -> Self {
|
||||||
|
Self {
|
||||||
|
value,
|
||||||
|
timestamp: Utc::now(),
|
||||||
|
node_id,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get(&self) -> &T {
|
||||||
|
&self.value
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set(&mut self, value: T, node_id: NodeId) {
|
||||||
|
self.value = value;
|
||||||
|
self.timestamp = Utc::now();
|
||||||
|
self.node_id = node_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn apply_lww(&mut self, value: T, timestamp: DateTime<Utc>, node_id: NodeId) {
|
||||||
|
if timestamp > self.timestamp || (timestamp == self.timestamp && node_id > self.node_id) {
|
||||||
|
self.value = value;
|
||||||
|
self.timestamp = timestamp;
|
||||||
|
self.node_id = node_id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn merge(&mut self, other: &Self) {
|
||||||
|
self.apply_lww(other.value.clone(), other.timestamp, other.node_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allow transparent access to the inner value
|
||||||
|
impl<T: Clone> Deref for SyncedValue<T> {
|
||||||
|
type Target = T;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Clone> DerefMut for SyncedValue<T> {
|
||||||
|
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||||
|
&mut self.value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wrapper for a sync message that goes over gossip
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SyncMessage<T> {
|
||||||
|
/// Unique message ID
|
||||||
|
pub message_id: String,
|
||||||
|
/// Node that sent this
|
||||||
|
pub node_id: NodeId,
|
||||||
|
/// When it was sent
|
||||||
|
pub timestamp: DateTime<Utc>,
|
||||||
|
/// The actual sync operation
|
||||||
|
pub operation: T,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Serialize> SyncMessage<T> {
|
||||||
|
pub fn new(node_id: NodeId, operation: T) -> Self {
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||||
|
let seq = COUNTER.fetch_add(1, Ordering::SeqCst);
|
||||||
|
|
||||||
|
Self {
|
||||||
|
message_id: format!("{}-{}-{}", node_id, Utc::now().timestamp_millis(), seq),
|
||||||
|
node_id,
|
||||||
|
timestamp: Utc::now(),
|
||||||
|
operation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
|
||||||
|
Ok(serde_json::to_vec(self)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: for<'de> Deserialize<'de>> SyncMessage<T> {
|
||||||
|
pub fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self> {
|
||||||
|
Ok(serde_json::from_slice(bytes)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper trait for types that can be synced
|
||||||
|
pub trait Syncable: Sized {
|
||||||
|
type Operation: Serialize + for<'de> Deserialize<'de> + Clone;
|
||||||
|
|
||||||
|
/// Apply a sync operation to this value
|
||||||
|
fn apply_sync_op(&mut self, op: &Self::Operation);
|
||||||
|
|
||||||
|
/// Get the node ID for this instance
|
||||||
|
fn node_id(&self) -> &NodeId;
|
||||||
|
|
||||||
|
/// Create a sync message for an operation
|
||||||
|
fn create_sync_message(&self, op: Self::Operation) -> SyncMessage<Self::Operation> {
|
||||||
|
SyncMessage::new(self.node_id().clone(), op)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_synced_value() {
|
||||||
|
let mut val = SyncedValue::new(42, "node1".to_string());
|
||||||
|
assert_eq!(*val.get(), 42);
|
||||||
|
|
||||||
|
val.set(100, "node1".to_string());
|
||||||
|
assert_eq!(*val.get(), 100);
|
||||||
|
|
||||||
|
// Test LWW semantics
|
||||||
|
let old_time = Utc::now() - chrono::Duration::seconds(10);
|
||||||
|
val.apply_lww(50, old_time, "node2".to_string());
|
||||||
|
assert_eq!(*val.get(), 100); // Should not update with older timestamp
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sync_message() {
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
struct TestOp {
|
||||||
|
value: i32,
|
||||||
|
}
|
||||||
|
|
||||||
|
let op = TestOp { value: 42 };
|
||||||
|
let msg = SyncMessage::new("node1".to_string(), op);
|
||||||
|
|
||||||
|
let bytes = msg.to_bytes().unwrap();
|
||||||
|
let decoded = SyncMessage::<TestOp>::from_bytes(&bytes).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(decoded.node_id, "node1");
|
||||||
|
assert_eq!(decoded.operation.value, 42);
|
||||||
|
}
|
||||||
|
}
|
||||||
98
crates/lib/tests/our_messages_test.rs
Normal file
98
crates/lib/tests/our_messages_test.rs
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
use lib::{ChatDb, Result};
|
||||||
|
use chrono::Datelike;
|
||||||
|
|
||||||
|
/// Test that we can get messages from the Dutch phone number conversation
|
||||||
|
#[test]
|
||||||
|
fn test_get_our_messages_default_range() -> Result<()> {
|
||||||
|
let db = ChatDb::open("chat.db")?;
|
||||||
|
|
||||||
|
// Get messages from January 2024 to now (default)
|
||||||
|
let messages = db.get_our_messages(None, None)?;
|
||||||
|
|
||||||
|
println!("Found {} messages from January 2024 to now", messages.len());
|
||||||
|
|
||||||
|
// Verify we got some messages
|
||||||
|
assert!(messages.len() > 0, "Should find messages in the conversation");
|
||||||
|
|
||||||
|
// Verify messages are in chronological order (ASC)
|
||||||
|
for i in 1..messages.len().min(10) {
|
||||||
|
if let (Some(prev_date), Some(curr_date)) = (messages[i-1].date, messages[i].date) {
|
||||||
|
assert!(
|
||||||
|
prev_date <= curr_date,
|
||||||
|
"Messages should be in ascending date order"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify all messages are from 2024 or later
|
||||||
|
for msg in messages.iter().take(10) {
|
||||||
|
if let Some(date) = msg.date {
|
||||||
|
assert!(date.year() >= 2024, "Messages should be from 2024 or later");
|
||||||
|
println!("Message date: {}, from_me: {}, text: {:?}",
|
||||||
|
date, msg.is_from_me, msg.text.as_ref().map(|s| &s[..s.len().min(50)]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test that we can get messages with a custom date range
|
||||||
|
#[test]
|
||||||
|
fn test_get_our_messages_custom_range() -> Result<()> {
|
||||||
|
use chrono::{TimeZone, Utc};
|
||||||
|
|
||||||
|
let db = ChatDb::open("chat.db")?;
|
||||||
|
|
||||||
|
// Get messages from March 2024 to June 2024
|
||||||
|
let start = Utc.with_ymd_and_hms(2024, 3, 1, 0, 0, 0).unwrap();
|
||||||
|
let end = Utc.with_ymd_and_hms(2024, 6, 1, 0, 0, 0).unwrap();
|
||||||
|
|
||||||
|
let messages = db.get_our_messages(Some(start), Some(end))?;
|
||||||
|
|
||||||
|
println!("Found {} messages from March to June 2024", messages.len());
|
||||||
|
|
||||||
|
// Verify all messages are within the date range
|
||||||
|
for msg in &messages {
|
||||||
|
if let Some(date) = msg.date {
|
||||||
|
assert!(
|
||||||
|
date >= start && date <= end,
|
||||||
|
"Message date {} should be between {} and {}",
|
||||||
|
date, start, end
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test displaying a summary of the conversation
|
||||||
|
#[test]
|
||||||
|
fn test_conversation_summary() -> Result<()> {
|
||||||
|
let db = ChatDb::open("chat.db")?;
|
||||||
|
|
||||||
|
let messages = db.get_our_messages(None, None)?;
|
||||||
|
|
||||||
|
println!("\n=== Conversation Summary ===");
|
||||||
|
println!("Total messages: {}", messages.len());
|
||||||
|
|
||||||
|
let from_me = messages.iter().filter(|m| m.is_from_me).count();
|
||||||
|
let from_them = messages.len() - from_me;
|
||||||
|
|
||||||
|
println!("From me: {}", from_me);
|
||||||
|
println!("From them: {}", from_them);
|
||||||
|
|
||||||
|
// Show first few messages
|
||||||
|
println!("\nFirst 5 messages:");
|
||||||
|
for (i, msg) in messages.iter().take(5).enumerate() {
|
||||||
|
if let Some(date) = msg.date {
|
||||||
|
let sender = if msg.is_from_me { "Me" } else { "Them" };
|
||||||
|
let text = msg.text.as_ref()
|
||||||
|
.map(|t| if t.len() > 60 { format!("{}...", &t[..60]) } else { t.clone() })
|
||||||
|
.unwrap_or_else(|| "[No text]".to_string());
|
||||||
|
|
||||||
|
println!("{}. {} ({}): {}", i + 1, date.format("%Y-%m-%d %H:%M"), sender, text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
157
crates/lib/tests/sync_integration.rs
Normal file
157
crates/lib/tests/sync_integration.rs
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
use lib::sync::{synced, SyncMessage, Syncable};
|
||||||
|
use iroh::{Endpoint, protocol::{Router, ProtocolHandler, AcceptError}};
|
||||||
|
use anyhow::Result;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
|
/// Test configuration that can be synced
|
||||||
|
#[synced]
|
||||||
|
struct TestConfig {
|
||||||
|
value: i32,
|
||||||
|
name: String,
|
||||||
|
|
||||||
|
#[sync(skip)]
|
||||||
|
node_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ALPN identifier for our sync protocol
|
||||||
|
const SYNC_ALPN: &[u8] = b"/lonni/sync/1";
|
||||||
|
|
||||||
|
/// Protocol handler for receiving sync messages
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct SyncProtocol {
|
||||||
|
config: Arc<Mutex<TestConfig>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProtocolHandler for SyncProtocol {
|
||||||
|
async fn accept(&self, connection: iroh::endpoint::Connection) -> Result<(), AcceptError> {
|
||||||
|
println!("Accepting connection from: {}", connection.remote_id());
|
||||||
|
|
||||||
|
// Accept the bidirectional stream
|
||||||
|
let (mut send, mut recv) = connection.accept_bi().await
|
||||||
|
.map_err(AcceptError::from_err)?;
|
||||||
|
|
||||||
|
println!("Stream accepted, reading message...");
|
||||||
|
|
||||||
|
// Read the sync message
|
||||||
|
let bytes = recv.read_to_end(1024 * 1024).await
|
||||||
|
.map_err(AcceptError::from_err)?;
|
||||||
|
|
||||||
|
println!("Received {} bytes", bytes.len());
|
||||||
|
|
||||||
|
// Deserialize and apply
|
||||||
|
let msg = SyncMessage::<TestConfigOp>::from_bytes(&bytes)
|
||||||
|
.map_err(|e| AcceptError::from_err(std::io::Error::new(std::io::ErrorKind::InvalidData, e)))?;
|
||||||
|
|
||||||
|
println!("Applying operation from node: {}", msg.node_id);
|
||||||
|
|
||||||
|
let mut config = self.config.lock().await;
|
||||||
|
config.apply_op(&msg.operation);
|
||||||
|
|
||||||
|
println!("Operation applied successfully");
|
||||||
|
|
||||||
|
// Close the stream
|
||||||
|
send.finish()
|
||||||
|
.map_err(AcceptError::from_err)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn test_sync_between_two_nodes() -> Result<()> {
|
||||||
|
println!("\n=== Testing Sync Between Two Nodes ===\n");
|
||||||
|
|
||||||
|
// Create two endpoints
|
||||||
|
let node1 = Endpoint::builder().bind().await?;
|
||||||
|
let node2 = Endpoint::builder().bind().await?;
|
||||||
|
|
||||||
|
let node1_addr = node1.addr();
|
||||||
|
let node2_addr = node2.addr();
|
||||||
|
|
||||||
|
let node1_id = node1_addr.id.to_string();
|
||||||
|
let node2_id = node2_addr.id.to_string();
|
||||||
|
|
||||||
|
println!("Node 1: {}", node1_id);
|
||||||
|
println!("Node 2: {}", node2_id);
|
||||||
|
|
||||||
|
// Create synced configs on both nodes
|
||||||
|
let mut config1 = TestConfig::new(
|
||||||
|
42,
|
||||||
|
"initial".to_string(),
|
||||||
|
node1_id.clone(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let config2 = TestConfig::new(
|
||||||
|
42,
|
||||||
|
"initial".to_string(),
|
||||||
|
node2_id.clone(),
|
||||||
|
);
|
||||||
|
let config2_shared = Arc::new(Mutex::new(config2));
|
||||||
|
|
||||||
|
println!("\nInitial state:");
|
||||||
|
println!(" Node 1: value={}, name={}", config1.value(), config1.name());
|
||||||
|
{
|
||||||
|
let config2 = config2_shared.lock().await;
|
||||||
|
println!(" Node 2: value={}, name={}", config2.value(), config2.name());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set up router on node2 to accept incoming connections
|
||||||
|
println!("\nSetting up node2 router...");
|
||||||
|
let protocol = SyncProtocol {
|
||||||
|
config: config2_shared.clone(),
|
||||||
|
};
|
||||||
|
let router = Router::builder(node2)
|
||||||
|
.accept(SYNC_ALPN, protocol)
|
||||||
|
.spawn();
|
||||||
|
|
||||||
|
router.endpoint().online().await;
|
||||||
|
println!("✓ Node2 router ready");
|
||||||
|
|
||||||
|
// Node 1 changes the value
|
||||||
|
println!("\nNode 1 changing value to 100...");
|
||||||
|
let op = config1.set_value(100);
|
||||||
|
|
||||||
|
// Serialize the operation
|
||||||
|
let sync_msg = SyncMessage::new(node1_id.clone(), op);
|
||||||
|
let bytes = sync_msg.to_bytes()?;
|
||||||
|
println!("Serialized to {} bytes", bytes.len());
|
||||||
|
|
||||||
|
// Establish QUIC connection from node1 to node2
|
||||||
|
println!("\nEstablishing QUIC connection...");
|
||||||
|
let conn = node1.connect(node2_addr.clone(), SYNC_ALPN).await?;
|
||||||
|
println!("✓ Connection established");
|
||||||
|
|
||||||
|
// Open a bidirectional stream
|
||||||
|
let (mut send, _recv) = conn.open_bi().await?;
|
||||||
|
|
||||||
|
// Send the sync message
|
||||||
|
println!("Sending sync message...");
|
||||||
|
send.write_all(&bytes).await?;
|
||||||
|
send.finish()?;
|
||||||
|
println!("✓ Message sent");
|
||||||
|
|
||||||
|
// Wait a bit for the message to be processed
|
||||||
|
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
|
// Verify both configs have the same value
|
||||||
|
println!("\nFinal state:");
|
||||||
|
println!(" Node 1: value={}, name={}", config1.value(), config1.name());
|
||||||
|
{
|
||||||
|
let config2 = config2_shared.lock().await;
|
||||||
|
println!(" Node 2: value={}, name={}", config2.value(), config2.name());
|
||||||
|
|
||||||
|
assert_eq!(*config1.value(), 100);
|
||||||
|
assert_eq!(*config2.value(), 100);
|
||||||
|
assert_eq!(config1.name(), "initial");
|
||||||
|
assert_eq!(config2.name(), "initial");
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("\n✓ Sync successful!");
|
||||||
|
|
||||||
|
// Cleanup
|
||||||
|
router.shutdown().await?;
|
||||||
|
node1.close().await;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
58
crates/server/Cargo.toml
Normal file
58
crates/server/Cargo.toml
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
[package]
|
||||||
|
name = "server"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "server"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
# Bevy (headless)
|
||||||
|
bevy = { version = "0.17", default-features = false, features = [
|
||||||
|
"bevy_state",
|
||||||
|
] }
|
||||||
|
|
||||||
|
# Iroh - P2P networking and gossip
|
||||||
|
iroh = { workspace = true }
|
||||||
|
iroh-gossip = { workspace = true }
|
||||||
|
|
||||||
|
# Async runtime
|
||||||
|
tokio = { version = "1", features = ["full"] }
|
||||||
|
tokio-stream = "0.1"
|
||||||
|
futures-lite = "2.5"
|
||||||
|
|
||||||
|
# Database
|
||||||
|
rusqlite = { version = "0.37.0", features = ["bundled", "column_decltype", "load_extension"] }
|
||||||
|
|
||||||
|
# Serialization
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0"
|
||||||
|
toml = "0.9"
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
tracing = "0.1"
|
||||||
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|
||||||
|
# Error handling
|
||||||
|
thiserror = "2.0"
|
||||||
|
anyhow = "1.0"
|
||||||
|
|
||||||
|
# Date/time
|
||||||
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
|
|
||||||
|
# Random number generation
|
||||||
|
rand = "0.8"
|
||||||
|
|
||||||
|
# ML/AI - Candle for inference (using newer versions with better compatibility)
|
||||||
|
candle-core = "0.8"
|
||||||
|
candle-nn = "0.8"
|
||||||
|
candle-transformers = "0.8"
|
||||||
|
tokenizers = "0.20"
|
||||||
|
hf-hub = "0.3"
|
||||||
|
|
||||||
|
# Synchronization
|
||||||
|
parking_lot = { workspace = true }
|
||||||
|
|
||||||
|
# Local dependencies
|
||||||
|
lib = { path = "../lib" }
|
||||||
1
crates/server/src/assets/mod.rs
Normal file
1
crates/server/src/assets/mod.rs
Normal file
@@ -0,0 +1 @@
|
|||||||
|
// Asset loading and management will go here
|
||||||
14
crates/server/src/components/database.rs
Normal file
14
crates/server/src/components/database.rs
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
use bevy::prelude::*;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::config::Config;
|
||||||
|
|
||||||
|
/// Bevy resource wrapping application configuration
|
||||||
|
#[derive(Resource)]
|
||||||
|
pub struct AppConfig(pub Config);
|
||||||
|
|
||||||
|
/// Bevy resource wrapping database connection
|
||||||
|
#[derive(Resource)]
|
||||||
|
pub struct Database(pub Arc<Mutex<Connection>>);
|
||||||
87
crates/server/src/components/gossip.rs
Normal file
87
crates/server/src/components/gossip.rs
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
use bevy::prelude::*;
|
||||||
|
use iroh::protocol::Router;
|
||||||
|
use iroh::Endpoint;
|
||||||
|
use iroh_gossip::api::{GossipReceiver, GossipSender};
|
||||||
|
use iroh_gossip::net::Gossip;
|
||||||
|
use iroh_gossip::proto::TopicId;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
/// Message envelope for gossip sync
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SyncMessage {
|
||||||
|
/// The actual message from iMessage
|
||||||
|
pub message: lib::Message,
|
||||||
|
/// Timestamp when this was published to gossip
|
||||||
|
pub sync_timestamp: i64,
|
||||||
|
/// ID of the node that published this
|
||||||
|
pub publisher_node_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bevy resource wrapping the gossip handle
|
||||||
|
#[derive(Resource, Clone)]
|
||||||
|
pub struct IrohGossipHandle {
|
||||||
|
pub gossip: Gossip,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bevy resource wrapping the gossip sender
|
||||||
|
#[derive(Resource)]
|
||||||
|
pub struct IrohGossipSender {
|
||||||
|
pub sender: Arc<Mutex<GossipSender>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bevy resource wrapping the gossip receiver
|
||||||
|
#[derive(Resource)]
|
||||||
|
pub struct IrohGossipReceiver {
|
||||||
|
pub receiver: Arc<Mutex<GossipReceiver>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bevy resource with Iroh router
|
||||||
|
#[derive(Resource)]
|
||||||
|
pub struct IrohRouter {
|
||||||
|
pub router: Router,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bevy resource with Iroh endpoint
|
||||||
|
#[derive(Resource, Clone)]
|
||||||
|
pub struct IrohEndpoint {
|
||||||
|
pub endpoint: Endpoint,
|
||||||
|
pub node_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bevy resource for gossip topic ID
|
||||||
|
#[derive(Resource)]
|
||||||
|
pub struct GossipTopic(pub TopicId);
|
||||||
|
|
||||||
|
/// Bevy resource for tracking gossip initialization task
|
||||||
|
#[derive(Resource)]
|
||||||
|
pub struct GossipInitTask(pub bevy::tasks::Task<Option<(
|
||||||
|
Endpoint,
|
||||||
|
Gossip,
|
||||||
|
Router,
|
||||||
|
GossipSender,
|
||||||
|
GossipReceiver,
|
||||||
|
)>>);
|
||||||
|
|
||||||
|
/// Bevy message: a new message that needs to be published to gossip
|
||||||
|
#[derive(Message, Clone, Debug)]
|
||||||
|
pub struct PublishMessageEvent {
|
||||||
|
pub message: lib::Message,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bevy message: a message received from gossip that needs to be saved to SQLite
|
||||||
|
#[derive(Message, Clone, Debug)]
|
||||||
|
pub struct GossipMessageReceived {
|
||||||
|
pub sync_message: SyncMessage,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper to serialize a sync message
|
||||||
|
pub fn serialize_sync_message(msg: &SyncMessage) -> anyhow::Result<Vec<u8>> {
|
||||||
|
Ok(serde_json::to_vec(msg)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper to deserialize a sync message
|
||||||
|
pub fn deserialize_sync_message(data: &[u8]) -> anyhow::Result<SyncMessage> {
|
||||||
|
Ok(serde_json::from_slice(data)?)
|
||||||
|
}
|
||||||
5
crates/server/src/components/mod.rs
Normal file
5
crates/server/src/components/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
pub mod database;
|
||||||
|
pub mod gossip;
|
||||||
|
|
||||||
|
pub use database::*;
|
||||||
|
pub use gossip::*;
|
||||||
84
crates/server/src/config.rs
Normal file
84
crates/server/src/config.rs
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
use anyhow::{Context, Result};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::fs;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct Config {
|
||||||
|
pub database: DatabaseConfig,
|
||||||
|
pub services: ServicesConfig,
|
||||||
|
pub models: ModelsConfig,
|
||||||
|
pub tailscale: TailscaleConfig,
|
||||||
|
pub grpc: GrpcConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct DatabaseConfig {
|
||||||
|
pub path: String,
|
||||||
|
pub chat_db_path: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ServicesConfig {
|
||||||
|
pub poll_interval_ms: u64,
|
||||||
|
pub training_set_sample_rate: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ModelsConfig {
|
||||||
|
pub embedding_model: String,
|
||||||
|
pub emotion_model: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct TailscaleConfig {
|
||||||
|
pub hostname: String,
|
||||||
|
pub state_dir: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct GrpcConfig {
|
||||||
|
pub port: u16,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Config {
|
||||||
|
pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
|
||||||
|
let content = fs::read_to_string(path.as_ref())
|
||||||
|
.context(format!("Failed to read config file: {:?}", path.as_ref()))?;
|
||||||
|
let config: Config = toml::from_str(&content)
|
||||||
|
.context("Failed to parse config file")?;
|
||||||
|
Ok(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn default_config() -> Self {
|
||||||
|
Self {
|
||||||
|
database: DatabaseConfig {
|
||||||
|
path: "./us.db".to_string(),
|
||||||
|
chat_db_path: "./crates/lib/chat.db".to_string(),
|
||||||
|
},
|
||||||
|
services: ServicesConfig {
|
||||||
|
poll_interval_ms: 1000,
|
||||||
|
training_set_sample_rate: 0.05,
|
||||||
|
},
|
||||||
|
models: ModelsConfig {
|
||||||
|
embedding_model: "Qwen/Qwen3-Embedding-0.6B".to_string(),
|
||||||
|
emotion_model: "SamLowe/roberta-base-go_emotions".to_string(),
|
||||||
|
},
|
||||||
|
tailscale: TailscaleConfig {
|
||||||
|
hostname: "lonni-daemon".to_string(),
|
||||||
|
state_dir: "./tailscale-state".to_string(),
|
||||||
|
},
|
||||||
|
grpc: GrpcConfig {
|
||||||
|
port: 50051,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
||||||
|
let content = toml::to_string_pretty(self)
|
||||||
|
.context("Failed to serialize config")?;
|
||||||
|
fs::write(path.as_ref(), content)
|
||||||
|
.context(format!("Failed to write config file: {:?}", path.as_ref()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
5
crates/server/src/db/mod.rs
Normal file
5
crates/server/src/db/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
pub mod operations;
|
||||||
|
pub mod schema;
|
||||||
|
|
||||||
|
pub use operations::*;
|
||||||
|
pub use schema::*;
|
||||||
321
crates/server/src/db/operations.rs
Normal file
321
crates/server/src/db/operations.rs
Normal file
@@ -0,0 +1,321 @@
|
|||||||
|
use crate::db::schema::{deserialize_embedding, serialize_embedding};
|
||||||
|
use crate::models::*;
|
||||||
|
use chrono::{TimeZone, Utc};
|
||||||
|
use rusqlite::{params, Connection, OptionalExtension, Result, Row};
|
||||||
|
|
||||||
|
/// Insert a new message into the database
|
||||||
|
pub fn insert_message(conn: &Connection, msg: &lib::Message) -> Result<i64> {
|
||||||
|
let timestamp = msg.date.map(|dt| dt.timestamp());
|
||||||
|
let created_at = Utc::now().timestamp();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO messages (chat_db_rowid, text, timestamp, is_from_me, created_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||||
|
ON CONFLICT(chat_db_rowid) DO NOTHING",
|
||||||
|
params![msg.rowid, msg.text, timestamp, msg.is_from_me, created_at],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get message ID by chat.db rowid
|
||||||
|
pub fn get_message_id_by_chat_rowid(conn: &Connection, chat_db_rowid: i64) -> Result<Option<i64>> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id FROM messages WHERE chat_db_rowid = ?1",
|
||||||
|
params![chat_db_rowid],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get message by ID
|
||||||
|
pub fn get_message(conn: &Connection, id: i64) -> Result<Message> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, chat_db_rowid, text, timestamp, is_from_me, created_at FROM messages WHERE id = ?1",
|
||||||
|
params![id],
|
||||||
|
map_message_row,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn map_message_row(row: &Row) -> Result<Message> {
|
||||||
|
let timestamp: Option<i64> = row.get(3)?;
|
||||||
|
let created_at: i64 = row.get(5)?;
|
||||||
|
|
||||||
|
Ok(Message {
|
||||||
|
id: row.get(0)?,
|
||||||
|
chat_db_rowid: row.get(1)?,
|
||||||
|
text: row.get(2)?,
|
||||||
|
timestamp: timestamp.map(|ts| Utc.timestamp_opt(ts, 0).unwrap()),
|
||||||
|
is_from_me: row.get(4)?,
|
||||||
|
created_at: Utc.timestamp_opt(created_at, 0).unwrap(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert message embedding
|
||||||
|
pub fn insert_message_embedding(
|
||||||
|
conn: &Connection,
|
||||||
|
message_id: i64,
|
||||||
|
embedding: &[f32],
|
||||||
|
model_name: &str,
|
||||||
|
) -> Result<i64> {
|
||||||
|
let embedding_bytes = serialize_embedding(embedding);
|
||||||
|
let created_at = Utc::now().timestamp();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO message_embeddings (message_id, embedding, model_name, created_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4)",
|
||||||
|
params![message_id, embedding_bytes, model_name, created_at],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get message embedding
|
||||||
|
pub fn get_message_embedding(conn: &Connection, message_id: i64) -> Result<Option<MessageEmbedding>> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, message_id, embedding, model_name, created_at
|
||||||
|
FROM message_embeddings WHERE message_id = ?1",
|
||||||
|
params![message_id],
|
||||||
|
|row| {
|
||||||
|
let embedding_bytes: Vec<u8> = row.get(2)?;
|
||||||
|
let created_at: i64 = row.get(4)?;
|
||||||
|
|
||||||
|
Ok(MessageEmbedding {
|
||||||
|
id: row.get(0)?,
|
||||||
|
message_id: row.get(1)?,
|
||||||
|
embedding: deserialize_embedding(&embedding_bytes),
|
||||||
|
model_name: row.get(3)?,
|
||||||
|
created_at: Utc.timestamp_opt(created_at, 0).unwrap(),
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert or get word embedding
|
||||||
|
pub fn insert_word_embedding(
|
||||||
|
conn: &Connection,
|
||||||
|
word: &str,
|
||||||
|
embedding: &[f32],
|
||||||
|
model_name: &str,
|
||||||
|
) -> Result<i64> {
|
||||||
|
let embedding_bytes = serialize_embedding(embedding);
|
||||||
|
let created_at = Utc::now().timestamp();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO word_embeddings (word, embedding, model_name, created_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4)
|
||||||
|
ON CONFLICT(word) DO NOTHING",
|
||||||
|
params![word, embedding_bytes, model_name, created_at],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get word embedding
|
||||||
|
pub fn get_word_embedding(conn: &Connection, word: &str) -> Result<Option<WordEmbedding>> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, word, embedding, model_name, created_at
|
||||||
|
FROM word_embeddings WHERE word = ?1",
|
||||||
|
params![word],
|
||||||
|
|row| {
|
||||||
|
let embedding_bytes: Vec<u8> = row.get(2)?;
|
||||||
|
let created_at: i64 = row.get(4)?;
|
||||||
|
|
||||||
|
Ok(WordEmbedding {
|
||||||
|
id: row.get(0)?,
|
||||||
|
word: row.get(1)?,
|
||||||
|
embedding: deserialize_embedding(&embedding_bytes),
|
||||||
|
model_name: row.get(3)?,
|
||||||
|
created_at: Utc.timestamp_opt(created_at, 0).unwrap(),
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert emotion classification
|
||||||
|
pub fn insert_emotion(
|
||||||
|
conn: &Connection,
|
||||||
|
message_id: i64,
|
||||||
|
emotion: &str,
|
||||||
|
confidence: f64,
|
||||||
|
model_version: &str,
|
||||||
|
) -> Result<i64> {
|
||||||
|
let now = Utc::now().timestamp();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO emotions (message_id, emotion, confidence, model_version, created_at, updated_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||||
|
params![message_id, emotion, confidence, model_version, now, now],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Update emotion classification
|
||||||
|
pub fn update_emotion(
|
||||||
|
conn: &Connection,
|
||||||
|
message_id: i64,
|
||||||
|
emotion: &str,
|
||||||
|
confidence: f64,
|
||||||
|
) -> Result<()> {
|
||||||
|
let updated_at = Utc::now().timestamp();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE emotions SET emotion = ?1, confidence = ?2, updated_at = ?3
|
||||||
|
WHERE message_id = ?4",
|
||||||
|
params![emotion, confidence, updated_at, message_id],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get emotion by message ID
|
||||||
|
pub fn get_emotion_by_message_id(conn: &Connection, message_id: i64) -> Result<Option<Emotion>> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, message_id, emotion, confidence, model_version, created_at, updated_at
|
||||||
|
FROM emotions WHERE message_id = ?1",
|
||||||
|
params![message_id],
|
||||||
|
map_emotion_row,
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get emotion by ID
|
||||||
|
pub fn get_emotion_by_id(conn: &Connection, id: i64) -> Result<Option<Emotion>> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, message_id, emotion, confidence, model_version, created_at, updated_at
|
||||||
|
FROM emotions WHERE id = ?1",
|
||||||
|
params![id],
|
||||||
|
map_emotion_row,
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// List all emotions with optional filters
|
||||||
|
pub fn list_emotions(
|
||||||
|
conn: &Connection,
|
||||||
|
emotion_filter: Option<&str>,
|
||||||
|
min_confidence: Option<f64>,
|
||||||
|
limit: Option<i32>,
|
||||||
|
offset: Option<i32>,
|
||||||
|
) -> Result<Vec<Emotion>> {
|
||||||
|
let mut query = String::from(
|
||||||
|
"SELECT id, message_id, emotion, confidence, model_version, created_at, updated_at
|
||||||
|
FROM emotions WHERE 1=1"
|
||||||
|
);
|
||||||
|
|
||||||
|
if emotion_filter.is_some() {
|
||||||
|
query.push_str(" AND emotion = ?1");
|
||||||
|
}
|
||||||
|
|
||||||
|
if min_confidence.is_some() {
|
||||||
|
query.push_str(" AND confidence >= ?2");
|
||||||
|
}
|
||||||
|
|
||||||
|
query.push_str(" ORDER BY created_at DESC");
|
||||||
|
|
||||||
|
if limit.is_some() {
|
||||||
|
query.push_str(" LIMIT ?3");
|
||||||
|
}
|
||||||
|
|
||||||
|
if offset.is_some() {
|
||||||
|
query.push_str(" OFFSET ?4");
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut stmt = conn.prepare(&query)?;
|
||||||
|
let emotions = stmt
|
||||||
|
.query_map(
|
||||||
|
params![
|
||||||
|
emotion_filter.unwrap_or(""),
|
||||||
|
min_confidence.unwrap_or(0.0),
|
||||||
|
limit.unwrap_or(1000),
|
||||||
|
offset.unwrap_or(0),
|
||||||
|
],
|
||||||
|
map_emotion_row,
|
||||||
|
)?
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
|
||||||
|
Ok(emotions)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete emotion by ID
|
||||||
|
pub fn delete_emotion(conn: &Connection, id: i64) -> Result<()> {
|
||||||
|
conn.execute("DELETE FROM emotions WHERE id = ?1", params![id])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Count total emotions
|
||||||
|
pub fn count_emotions(conn: &Connection) -> Result<i32> {
|
||||||
|
conn.query_row("SELECT COUNT(*) FROM emotions", [], |row| row.get(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn map_emotion_row(row: &Row) -> Result<Emotion> {
|
||||||
|
let created_at: i64 = row.get(5)?;
|
||||||
|
let updated_at: i64 = row.get(6)?;
|
||||||
|
|
||||||
|
Ok(Emotion {
|
||||||
|
id: row.get(0)?,
|
||||||
|
message_id: row.get(1)?,
|
||||||
|
emotion: row.get(2)?,
|
||||||
|
confidence: row.get(3)?,
|
||||||
|
model_version: row.get(4)?,
|
||||||
|
created_at: Utc.timestamp_opt(created_at, 0).unwrap(),
|
||||||
|
updated_at: Utc.timestamp_opt(updated_at, 0).unwrap(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert emotion training sample
|
||||||
|
pub fn insert_training_sample(
|
||||||
|
conn: &Connection,
|
||||||
|
message_id: Option<i64>,
|
||||||
|
text: &str,
|
||||||
|
expected_emotion: &str,
|
||||||
|
) -> Result<i64> {
|
||||||
|
let now = Utc::now().timestamp();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO emotions_training_set (message_id, text, expected_emotion, created_at, updated_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||||
|
params![message_id, text, expected_emotion, now, now],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get state value from daemon_state table
|
||||||
|
pub fn get_state(conn: &Connection, key: &str) -> Result<Option<String>> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT value FROM daemon_state WHERE key = ?1",
|
||||||
|
params![key],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set state value in daemon_state table
|
||||||
|
pub fn set_state(conn: &Connection, key: &str, value: &str) -> Result<()> {
|
||||||
|
let updated_at = Utc::now().timestamp();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO daemon_state (key, value, updated_at)
|
||||||
|
VALUES (?1, ?2, ?3)
|
||||||
|
ON CONFLICT(key) DO UPDATE SET value = ?2, updated_at = ?3",
|
||||||
|
params![key, value, updated_at],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get last processed chat.db rowid from database or return 0
|
||||||
|
pub fn get_last_processed_rowid(conn: &Connection) -> Result<i64> {
|
||||||
|
Ok(get_state(conn, "last_processed_rowid")?
|
||||||
|
.and_then(|s| s.parse().ok())
|
||||||
|
.unwrap_or(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Save last processed chat.db rowid to database
|
||||||
|
pub fn save_last_processed_rowid(conn: &Connection, rowid: i64) -> Result<()> {
|
||||||
|
set_state(conn, "last_processed_rowid", &rowid.to_string())
|
||||||
|
}
|
||||||
207
crates/server/src/db/schema.rs
Normal file
207
crates/server/src/db/schema.rs
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
use rusqlite::{Connection, Result};
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
pub fn initialize_database(conn: &Connection) -> Result<()> {
|
||||||
|
info!("Initializing database schema");
|
||||||
|
|
||||||
|
// Load sqlite-vec extension (macOS only)
|
||||||
|
let vec_path = "./extensions/vec0.dylib";
|
||||||
|
|
||||||
|
// Try to load the vector extension (non-fatal if it fails for now)
|
||||||
|
match unsafe { conn.load_extension_enable() } {
|
||||||
|
Ok(_) => {
|
||||||
|
match unsafe { conn.load_extension(vec_path, None::<&str>) } {
|
||||||
|
Ok(_) => info!("Loaded sqlite-vec extension"),
|
||||||
|
Err(e) => info!("Could not load sqlite-vec extension: {}. Vector operations will not be available.", e),
|
||||||
|
}
|
||||||
|
let _ = unsafe { conn.load_extension_disable() };
|
||||||
|
}
|
||||||
|
Err(e) => info!("Extension loading not enabled: {}", e),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create messages table
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS messages (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
chat_db_rowid INTEGER UNIQUE NOT NULL,
|
||||||
|
text TEXT,
|
||||||
|
timestamp INTEGER,
|
||||||
|
is_from_me BOOLEAN NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL
|
||||||
|
)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create index on chat_db_rowid for fast lookups
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_messages_chat_db_rowid ON messages(chat_db_rowid)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create message_embeddings table
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS message_embeddings (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
message_id INTEGER NOT NULL,
|
||||||
|
embedding BLOB NOT NULL,
|
||||||
|
model_name TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE
|
||||||
|
)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create index on message_id
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_message_embeddings_message_id ON message_embeddings(message_id)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create word_embeddings table
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS word_embeddings (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
word TEXT UNIQUE NOT NULL,
|
||||||
|
embedding BLOB NOT NULL,
|
||||||
|
model_name TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL
|
||||||
|
)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create index on word
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_word_embeddings_word ON word_embeddings(word)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create emotions table
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS emotions (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
message_id INTEGER NOT NULL,
|
||||||
|
emotion TEXT NOT NULL,
|
||||||
|
confidence REAL NOT NULL,
|
||||||
|
model_version TEXT NOT NULL,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
updated_at INTEGER NOT NULL,
|
||||||
|
FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE
|
||||||
|
)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create indexes for emotions
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_emotions_message_id ON emotions(message_id)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_emotions_emotion ON emotions(emotion)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create emotions_training_set table
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS emotions_training_set (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
message_id INTEGER,
|
||||||
|
text TEXT NOT NULL,
|
||||||
|
expected_emotion TEXT NOT NULL,
|
||||||
|
actual_emotion TEXT,
|
||||||
|
confidence REAL,
|
||||||
|
is_validated BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
notes TEXT,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
updated_at INTEGER NOT NULL,
|
||||||
|
FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE SET NULL
|
||||||
|
)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create index on emotions_training_set
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_emotions_training_set_message_id ON emotions_training_set(message_id)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_emotions_training_set_validated ON emotions_training_set(is_validated)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create state table for daemon state persistence
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS daemon_state (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at INTEGER NOT NULL
|
||||||
|
)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create models table for storing ML model files
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS models (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
name TEXT UNIQUE NOT NULL,
|
||||||
|
model_type TEXT NOT NULL,
|
||||||
|
version TEXT NOT NULL,
|
||||||
|
file_data BLOB NOT NULL,
|
||||||
|
metadata TEXT,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
updated_at INTEGER NOT NULL
|
||||||
|
)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Create index on model name and type
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_models_name ON models(name)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_models_type ON models(model_type)",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
info!("Database schema initialized successfully");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper function to serialize f32 vector to bytes for storage
|
||||||
|
pub fn serialize_embedding(embedding: &[f32]) -> Vec<u8> {
|
||||||
|
embedding
|
||||||
|
.iter()
|
||||||
|
.flat_map(|f| f.to_le_bytes())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper function to deserialize bytes back to f32 vector
|
||||||
|
pub fn deserialize_embedding(bytes: &[u8]) -> Vec<f32> {
|
||||||
|
bytes
|
||||||
|
.chunks_exact(4)
|
||||||
|
.map(|chunk| {
|
||||||
|
let array: [u8; 4] = chunk.try_into().unwrap();
|
||||||
|
f32::from_le_bytes(array)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_embedding_serialization() {
|
||||||
|
let original = vec![1.0f32, 2.5, -3.7, 0.0, 100.5];
|
||||||
|
let serialized = serialize_embedding(&original);
|
||||||
|
let deserialized = deserialize_embedding(&serialized);
|
||||||
|
|
||||||
|
assert_eq!(original.len(), deserialized.len());
|
||||||
|
for (a, b) in original.iter().zip(deserialized.iter()) {
|
||||||
|
assert!((a - b).abs() < 1e-6);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
1
crates/server/src/entities/mod.rs
Normal file
1
crates/server/src/entities/mod.rs
Normal file
@@ -0,0 +1 @@
|
|||||||
|
// Entity builders and spawners will go here
|
||||||
42
crates/server/src/iroh_sync.rs
Normal file
42
crates/server/src/iroh_sync.rs
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use iroh::protocol::Router;
|
||||||
|
use iroh::Endpoint;
|
||||||
|
use iroh_gossip::api::{GossipReceiver, GossipSender};
|
||||||
|
use iroh_gossip::net::Gossip;
|
||||||
|
use iroh_gossip::proto::TopicId;
|
||||||
|
|
||||||
|
/// Initialize Iroh endpoint and gossip for the given topic
|
||||||
|
pub async fn init_iroh_gossip(
|
||||||
|
topic_id: TopicId,
|
||||||
|
) -> Result<(Endpoint, Gossip, Router, GossipSender, GossipReceiver)> {
|
||||||
|
println!("Initializing Iroh endpoint...");
|
||||||
|
|
||||||
|
// Create the Iroh endpoint
|
||||||
|
let endpoint = Endpoint::bind().await?;
|
||||||
|
println!("Endpoint created");
|
||||||
|
|
||||||
|
// Build the gossip protocol
|
||||||
|
println!("Building gossip protocol...");
|
||||||
|
let gossip = Gossip::builder().spawn(endpoint.clone());
|
||||||
|
|
||||||
|
// Setup the router to handle incoming connections
|
||||||
|
println!("Setting up router...");
|
||||||
|
let router = Router::builder(endpoint.clone())
|
||||||
|
.accept(iroh_gossip::ALPN, gossip.clone())
|
||||||
|
.spawn();
|
||||||
|
|
||||||
|
// Subscribe to the topic (no bootstrap peers for now)
|
||||||
|
println!("Subscribing to topic: {:?}", topic_id);
|
||||||
|
let bootstrap_peers = vec![];
|
||||||
|
let subscribe_handle = gossip.subscribe(topic_id, bootstrap_peers).await?;
|
||||||
|
|
||||||
|
// Split into sender and receiver
|
||||||
|
let (sender, mut receiver) = subscribe_handle.split();
|
||||||
|
|
||||||
|
// Wait for join to complete
|
||||||
|
println!("Waiting for gossip join...");
|
||||||
|
receiver.joined().await?;
|
||||||
|
println!("Gossip initialized successfully");
|
||||||
|
|
||||||
|
Ok((endpoint, gossip, router, sender, receiver))
|
||||||
|
}
|
||||||
96
crates/server/src/main.rs
Normal file
96
crates/server/src/main.rs
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
mod assets;
|
||||||
|
mod components;
|
||||||
|
mod config;
|
||||||
|
mod db;
|
||||||
|
mod entities;
|
||||||
|
mod iroh_sync;
|
||||||
|
mod models;
|
||||||
|
mod services;
|
||||||
|
mod systems;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use bevy::prelude::*;
|
||||||
|
use config::Config;
|
||||||
|
use iroh_gossip::proto::TopicId;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
// Re-export init function
|
||||||
|
pub use iroh_sync::init_iroh_gossip;
|
||||||
|
|
||||||
|
// Import components and systems
|
||||||
|
use components::*;
|
||||||
|
use systems::*;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
println!("Starting server");
|
||||||
|
|
||||||
|
// Load configuration and initialize database
|
||||||
|
let (config, us_db) = match initialize_app() {
|
||||||
|
Ok(data) => data,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("Failed to initialize app: {}", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Create a topic ID for gossip (use a fixed topic for now)
|
||||||
|
let mut topic_bytes = [0u8; 32];
|
||||||
|
topic_bytes[..10].copy_from_slice(b"us-sync-v1");
|
||||||
|
let topic_id = TopicId::from_bytes(topic_bytes);
|
||||||
|
|
||||||
|
// Start Bevy app (headless)
|
||||||
|
App::new()
|
||||||
|
.add_plugins(MinimalPlugins)
|
||||||
|
.add_message::<PublishMessageEvent>()
|
||||||
|
.add_message::<GossipMessageReceived>()
|
||||||
|
.insert_resource(AppConfig(config))
|
||||||
|
.insert_resource(Database(us_db))
|
||||||
|
.insert_resource(GossipTopic(topic_id))
|
||||||
|
.add_systems(Startup, (setup_database, setup_gossip))
|
||||||
|
.add_systems(
|
||||||
|
Update,
|
||||||
|
(
|
||||||
|
poll_gossip_init,
|
||||||
|
poll_chat_db,
|
||||||
|
detect_new_messages,
|
||||||
|
publish_to_gossip,
|
||||||
|
receive_from_gossip,
|
||||||
|
save_gossip_messages,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Initialize configuration and database
|
||||||
|
fn initialize_app() -> Result<(Config, Arc<Mutex<Connection>>)> {
|
||||||
|
let config = if Path::new("config.toml").exists() {
|
||||||
|
println!("Loading config from config.toml");
|
||||||
|
Config::from_file("config.toml")?
|
||||||
|
} else {
|
||||||
|
println!("No config.toml found, using default configuration");
|
||||||
|
let config = Config::default_config();
|
||||||
|
config
|
||||||
|
.save("config.toml")
|
||||||
|
.context("Failed to save default config")?;
|
||||||
|
println!("Saved default configuration to config.toml");
|
||||||
|
config
|
||||||
|
};
|
||||||
|
|
||||||
|
println!("Configuration loaded");
|
||||||
|
println!(" Database: {}", config.database.path);
|
||||||
|
println!(" Chat DB: {}", config.database.chat_db_path);
|
||||||
|
|
||||||
|
// Initialize database
|
||||||
|
println!("Initializing database at {}", config.database.path);
|
||||||
|
let conn =
|
||||||
|
Connection::open(&config.database.path).context("Failed to open database")?;
|
||||||
|
|
||||||
|
db::initialize_database(&conn).context("Failed to initialize database schema")?;
|
||||||
|
|
||||||
|
let us_db = Arc::new(Mutex::new(conn));
|
||||||
|
|
||||||
|
Ok((config, us_db))
|
||||||
|
}
|
||||||
60
crates/server/src/models.rs
Normal file
60
crates/server/src/models.rs
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Represents a message stored in our database
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct Message {
|
||||||
|
pub id: i64,
|
||||||
|
pub chat_db_rowid: i64,
|
||||||
|
pub text: Option<String>,
|
||||||
|
pub timestamp: Option<DateTime<Utc>>,
|
||||||
|
pub is_from_me: bool,
|
||||||
|
pub created_at: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Represents a message embedding (full message vector)
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct MessageEmbedding {
|
||||||
|
pub id: i64,
|
||||||
|
pub message_id: i64,
|
||||||
|
pub embedding: Vec<f32>,
|
||||||
|
pub model_name: String,
|
||||||
|
pub created_at: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Represents a word embedding
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct WordEmbedding {
|
||||||
|
pub id: i64,
|
||||||
|
pub word: String,
|
||||||
|
pub embedding: Vec<f32>,
|
||||||
|
pub model_name: String,
|
||||||
|
pub created_at: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Represents an emotion classification for a message
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct Emotion {
|
||||||
|
pub id: i64,
|
||||||
|
pub message_id: i64,
|
||||||
|
pub emotion: String,
|
||||||
|
pub confidence: f64,
|
||||||
|
pub model_version: String,
|
||||||
|
pub created_at: DateTime<Utc>,
|
||||||
|
pub updated_at: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Represents an emotion training sample
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct EmotionTrainingSample {
|
||||||
|
pub id: i64,
|
||||||
|
pub message_id: Option<i64>,
|
||||||
|
pub text: String,
|
||||||
|
pub expected_emotion: String,
|
||||||
|
pub actual_emotion: Option<String>,
|
||||||
|
pub confidence: Option<f64>,
|
||||||
|
pub is_validated: bool,
|
||||||
|
pub notes: Option<String>,
|
||||||
|
pub created_at: DateTime<Utc>,
|
||||||
|
pub updated_at: DateTime<Utc>,
|
||||||
|
}
|
||||||
72
crates/server/src/proto/emotions.proto
Normal file
72
crates/server/src/proto/emotions.proto
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
package emotions;
|
||||||
|
|
||||||
|
// Emotion classification for a message
|
||||||
|
message Emotion {
|
||||||
|
int64 id = 1;
|
||||||
|
int64 message_id = 2;
|
||||||
|
string emotion = 3;
|
||||||
|
double confidence = 4;
|
||||||
|
string model_version = 5;
|
||||||
|
int64 created_at = 6;
|
||||||
|
int64 updated_at = 7;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request to get a single emotion by message ID
|
||||||
|
message GetEmotionRequest {
|
||||||
|
int64 message_id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request to get multiple emotions with optional filters
|
||||||
|
message GetEmotionsRequest {
|
||||||
|
repeated int64 message_ids = 1;
|
||||||
|
optional string emotion_filter = 2;
|
||||||
|
optional double min_confidence = 3;
|
||||||
|
optional int32 limit = 4;
|
||||||
|
optional int32 offset = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Response containing multiple emotions
|
||||||
|
message EmotionsResponse {
|
||||||
|
repeated Emotion emotions = 1;
|
||||||
|
int32 total_count = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request to update an emotion (for corrections/fine-tuning)
|
||||||
|
message UpdateEmotionRequest {
|
||||||
|
int64 message_id = 1;
|
||||||
|
string emotion = 2;
|
||||||
|
double confidence = 3;
|
||||||
|
optional string notes = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request to delete an emotion
|
||||||
|
message DeleteEmotionRequest {
|
||||||
|
int64 id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generic response for mutations
|
||||||
|
message EmotionResponse {
|
||||||
|
bool success = 1;
|
||||||
|
string message = 2;
|
||||||
|
optional Emotion emotion = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty message for list all
|
||||||
|
message Empty {}
|
||||||
|
|
||||||
|
// The emotion service with full CRUD operations
|
||||||
|
service EmotionService {
|
||||||
|
// Read operations
|
||||||
|
rpc GetEmotion(GetEmotionRequest) returns (Emotion);
|
||||||
|
rpc GetEmotions(GetEmotionsRequest) returns (EmotionsResponse);
|
||||||
|
rpc ListAllEmotions(Empty) returns (EmotionsResponse);
|
||||||
|
|
||||||
|
// Update operations (for classification corrections and fine-tuning)
|
||||||
|
rpc UpdateEmotion(UpdateEmotionRequest) returns (EmotionResponse);
|
||||||
|
rpc BatchUpdateEmotions(stream UpdateEmotionRequest) returns (EmotionResponse);
|
||||||
|
|
||||||
|
// Delete operation
|
||||||
|
rpc DeleteEmotion(DeleteEmotionRequest) returns (EmotionResponse);
|
||||||
|
}
|
||||||
121
crates/server/src/services/chat_poller.rs
Normal file
121
crates/server/src/services/chat_poller.rs
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
use crate::db;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use chrono::Utc;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::sync::{mpsc, Mutex};
|
||||||
|
use tokio::time;
|
||||||
|
use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
|
pub struct ChatPollerService {
|
||||||
|
chat_db_path: String,
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
tx: mpsc::Sender<lib::Message>,
|
||||||
|
poll_interval: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatPollerService {
|
||||||
|
pub fn new(
|
||||||
|
chat_db_path: String,
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
tx: mpsc::Sender<lib::Message>,
|
||||||
|
poll_interval_ms: u64,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
chat_db_path,
|
||||||
|
us_db,
|
||||||
|
tx,
|
||||||
|
poll_interval: Duration::from_millis(poll_interval_ms),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run(&self) -> Result<()> {
|
||||||
|
info!("Starting chat poller service");
|
||||||
|
info!("Polling {} every {:?}", self.chat_db_path, self.poll_interval);
|
||||||
|
|
||||||
|
// Get last processed rowid from database
|
||||||
|
let us_db = self.us_db.lock().await;
|
||||||
|
let mut last_rowid = db::get_last_processed_rowid(&us_db)
|
||||||
|
.context("Failed to get last processed rowid")?;
|
||||||
|
drop(us_db);
|
||||||
|
|
||||||
|
info!("Starting from rowid: {}", last_rowid);
|
||||||
|
|
||||||
|
let mut interval = time::interval(self.poll_interval);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
interval.tick().await;
|
||||||
|
|
||||||
|
match self.poll_messages(last_rowid).await {
|
||||||
|
Ok(new_messages) => {
|
||||||
|
if !new_messages.is_empty() {
|
||||||
|
info!("Found {} new messages", new_messages.len());
|
||||||
|
|
||||||
|
for msg in new_messages {
|
||||||
|
// Update last_rowid
|
||||||
|
if msg.rowid > last_rowid {
|
||||||
|
last_rowid = msg.rowid;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send message to processing pipeline
|
||||||
|
if let Err(e) = self.tx.send(msg).await {
|
||||||
|
error!("Failed to send message to processing pipeline: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save state to database
|
||||||
|
let us_db = self.us_db.lock().await;
|
||||||
|
if let Err(e) = db::save_last_processed_rowid(&us_db, last_rowid) {
|
||||||
|
warn!("Failed to save last processed rowid: {}", e);
|
||||||
|
}
|
||||||
|
drop(us_db);
|
||||||
|
} else {
|
||||||
|
debug!("No new messages");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Error polling messages: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn poll_messages(&self, last_rowid: i64) -> Result<Vec<lib::Message>> {
|
||||||
|
// Check if chat.db exists
|
||||||
|
if !Path::new(&self.chat_db_path).exists() {
|
||||||
|
return Err(anyhow::anyhow!("chat.db not found at {}", self.chat_db_path));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open chat.db (read-only)
|
||||||
|
let chat_db = lib::ChatDb::open(&self.chat_db_path)
|
||||||
|
.context("Failed to open chat.db")?;
|
||||||
|
|
||||||
|
// Get messages with rowid > last_rowid
|
||||||
|
// We'll use the existing get_our_messages but need to filter by rowid
|
||||||
|
// For now, let's get recent messages and filter in-memory
|
||||||
|
let start_date = Some(Utc::now() - chrono::Duration::days(7));
|
||||||
|
let end_date = Some(Utc::now());
|
||||||
|
|
||||||
|
let messages = chat_db
|
||||||
|
.get_our_messages(start_date, end_date)
|
||||||
|
.context("Failed to get messages from chat.db")?;
|
||||||
|
|
||||||
|
// Filter messages with rowid > last_rowid and ensure they're not duplicates
|
||||||
|
let new_messages: Vec<lib::Message> = messages
|
||||||
|
.into_iter()
|
||||||
|
.filter(|msg| msg.rowid > last_rowid)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Insert new messages into our database
|
||||||
|
let us_db = self.us_db.lock().await;
|
||||||
|
for msg in &new_messages {
|
||||||
|
if let Err(e) = db::insert_message(&us_db, msg) {
|
||||||
|
warn!("Failed to insert message {}: {}", msg.rowid, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(new_messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
110
crates/server/src/services/embedding_service.rs
Normal file
110
crates/server/src/services/embedding_service.rs
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
use crate::db;
|
||||||
|
use anyhow::Result;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::{mpsc, Mutex};
|
||||||
|
use tracing::{error, info, warn};
|
||||||
|
|
||||||
|
/// Service responsible for generating embeddings for messages and words
|
||||||
|
pub struct EmbeddingService {
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
rx: mpsc::Receiver<lib::Message>,
|
||||||
|
model_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EmbeddingService {
|
||||||
|
pub fn new(
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
rx: mpsc::Receiver<lib::Message>,
|
||||||
|
model_name: String,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
us_db,
|
||||||
|
rx,
|
||||||
|
model_name,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run(mut self) -> Result<()> {
|
||||||
|
info!("Starting embedding service with model: {}", self.model_name);
|
||||||
|
|
||||||
|
// TODO: Load the embedding model here
|
||||||
|
// For now, we'll create a placeholder implementation
|
||||||
|
info!("Loading embedding model...");
|
||||||
|
// let model = load_embedding_model(&self.model_name)?;
|
||||||
|
info!("Embedding model loaded (placeholder)");
|
||||||
|
|
||||||
|
while let Some(msg) = self.rx.recv().await {
|
||||||
|
if let Err(e) = self.process_message(&msg).await {
|
||||||
|
error!("Error processing message {}: {}", msg.rowid, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process_message(&self, msg: &lib::Message) -> Result<()> {
|
||||||
|
// Get message ID from our database
|
||||||
|
let us_db = self.us_db.lock().await;
|
||||||
|
let message_id = match db::get_message_id_by_chat_rowid(&us_db, msg.rowid)? {
|
||||||
|
Some(id) => id,
|
||||||
|
None => {
|
||||||
|
warn!("Message {} not found in database, skipping", msg.rowid);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Check if embedding already exists
|
||||||
|
if db::get_message_embedding(&us_db, message_id)?.is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip if message has no text
|
||||||
|
let text = match &msg.text {
|
||||||
|
Some(t) if !t.is_empty() => t,
|
||||||
|
_ => return Ok(()),
|
||||||
|
};
|
||||||
|
|
||||||
|
drop(us_db);
|
||||||
|
|
||||||
|
// Generate embedding for the full message
|
||||||
|
// TODO: Replace with actual model inference
|
||||||
|
let message_embedding = self.generate_embedding(text)?;
|
||||||
|
|
||||||
|
// Store message embedding
|
||||||
|
let us_db = self.us_db.lock().await;
|
||||||
|
db::insert_message_embedding(&us_db, message_id, &message_embedding, &self.model_name)?;
|
||||||
|
|
||||||
|
// Tokenize and generate word embeddings
|
||||||
|
let words = self.tokenize(text);
|
||||||
|
for word in words {
|
||||||
|
// Check if word embedding exists
|
||||||
|
if db::get_word_embedding(&us_db, &word)?.is_none() {
|
||||||
|
// Generate embedding for word
|
||||||
|
let word_embedding = self.generate_embedding(&word)?;
|
||||||
|
db::insert_word_embedding(&us_db, &word, &word_embedding, &self.model_name)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(us_db);
|
||||||
|
info!("Generated embeddings for message {}", msg.rowid);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generate_embedding(&self, text: &str) -> Result<Vec<f32>> {
|
||||||
|
// TODO: Replace with actual model inference using Candle
|
||||||
|
// For now, return a placeholder embedding of dimension 1024
|
||||||
|
let embedding = vec![0.0f32; 1024];
|
||||||
|
Ok(embedding)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tokenize(&self, text: &str) -> Vec<String> {
|
||||||
|
// Simple word tokenization (split on whitespace and punctuation)
|
||||||
|
// TODO: Replace with proper tokenizer
|
||||||
|
text.split(|c: char| c.is_whitespace() || c.is_ascii_punctuation())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(|s| s.to_lowercase())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
119
crates/server/src/services/emotion_service.rs
Normal file
119
crates/server/src/services/emotion_service.rs
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
use crate::db;
|
||||||
|
use anyhow::Result;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::{mpsc, Mutex};
|
||||||
|
use tracing::{error, info, warn};
|
||||||
|
|
||||||
|
/// Service responsible for classifying emotions in messages
|
||||||
|
pub struct EmotionService {
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
rx: mpsc::Receiver<lib::Message>,
|
||||||
|
model_version: String,
|
||||||
|
training_sample_rate: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EmotionService {
|
||||||
|
pub fn new(
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
rx: mpsc::Receiver<lib::Message>,
|
||||||
|
model_version: String,
|
||||||
|
training_sample_rate: f64,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
us_db,
|
||||||
|
rx,
|
||||||
|
model_version,
|
||||||
|
training_sample_rate,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run(mut self) -> Result<()> {
|
||||||
|
info!(
|
||||||
|
"Starting emotion classification service with model: {}",
|
||||||
|
self.model_version
|
||||||
|
);
|
||||||
|
info!(
|
||||||
|
"Training sample rate: {:.2}%",
|
||||||
|
self.training_sample_rate * 100.0
|
||||||
|
);
|
||||||
|
|
||||||
|
// TODO: Load the RoBERTa emotion classification model here
|
||||||
|
info!("Loading RoBERTa-base-go_emotions model...");
|
||||||
|
// let model = load_emotion_model(&self.model_version)?;
|
||||||
|
info!("Emotion model loaded (placeholder)");
|
||||||
|
|
||||||
|
while let Some(msg) = self.rx.recv().await {
|
||||||
|
if let Err(e) = self.process_message(&msg).await {
|
||||||
|
error!("Error processing message {}: {}", msg.rowid, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process_message(&self, msg: &lib::Message) -> Result<()> {
|
||||||
|
// Get message ID from our database
|
||||||
|
let us_db = self.us_db.lock().await;
|
||||||
|
let message_id = match db::get_message_id_by_chat_rowid(&us_db, msg.rowid)? {
|
||||||
|
Some(id) => id,
|
||||||
|
None => {
|
||||||
|
warn!("Message {} not found in database, skipping", msg.rowid);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Check if emotion classification already exists
|
||||||
|
if db::get_emotion_by_message_id(&us_db, message_id)?.is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip if message has no text
|
||||||
|
let text = match &msg.text {
|
||||||
|
Some(t) if !t.is_empty() => t,
|
||||||
|
_ => return Ok(()),
|
||||||
|
};
|
||||||
|
|
||||||
|
drop(us_db);
|
||||||
|
|
||||||
|
// Classify emotion
|
||||||
|
// TODO: Replace with actual model inference
|
||||||
|
let (emotion, confidence) = self.classify_emotion(text)?;
|
||||||
|
|
||||||
|
// Store emotion classification
|
||||||
|
let us_db = self.us_db.lock().await;
|
||||||
|
db::insert_emotion(&us_db, message_id, &emotion, confidence, &self.model_version)?;
|
||||||
|
|
||||||
|
// Randomly add to training set based on sample rate
|
||||||
|
if rand::random::<f64>() < self.training_sample_rate {
|
||||||
|
db::insert_training_sample(&us_db, Some(message_id), text, &emotion)?;
|
||||||
|
info!(
|
||||||
|
"Added message {} to training set (emotion: {})",
|
||||||
|
msg.rowid, emotion
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(us_db);
|
||||||
|
info!(
|
||||||
|
"Classified message {} as {} (confidence: {:.2})",
|
||||||
|
msg.rowid, emotion, confidence
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_emotion(&self, text: &str) -> Result<(String, f64)> {
|
||||||
|
// TODO: Replace with actual RoBERTa-base-go_emotions inference using Candle
|
||||||
|
// The model outputs probabilities for 28 emotions:
|
||||||
|
// admiration, amusement, anger, annoyance, approval, caring, confusion,
|
||||||
|
// curiosity, desire, disappointment, disapproval, disgust, embarrassment,
|
||||||
|
// excitement, fear, gratitude, grief, joy, love, nervousness, optimism,
|
||||||
|
// pride, realization, relief, remorse, sadness, surprise, neutral
|
||||||
|
|
||||||
|
// For now, return a placeholder
|
||||||
|
let emotion = "neutral".to_string();
|
||||||
|
let confidence = 0.85;
|
||||||
|
|
||||||
|
Ok((emotion, confidence))
|
||||||
|
}
|
||||||
|
}
|
||||||
232
crates/server/src/services/grpc_server.rs
Normal file
232
crates/server/src/services/grpc_server.rs
Normal file
@@ -0,0 +1,232 @@
|
|||||||
|
use crate::db;
|
||||||
|
use anyhow::Result;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
use tonic::{Request, Response, Status};
|
||||||
|
use tracing::{error, info};
|
||||||
|
|
||||||
|
// Include the generated protobuf code
|
||||||
|
pub mod emotions {
|
||||||
|
tonic::include_proto!("emotions");
|
||||||
|
}
|
||||||
|
|
||||||
|
use emotions::emotion_service_server::{EmotionService as EmotionServiceTrait, EmotionServiceServer};
|
||||||
|
use emotions::*;
|
||||||
|
|
||||||
|
pub struct GrpcServer {
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
address: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GrpcServer {
|
||||||
|
pub fn new(us_db: Arc<Mutex<Connection>>, address: String) -> Self {
|
||||||
|
Self { us_db, address }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run(self) -> Result<()> {
|
||||||
|
let addr = self.address.parse()?;
|
||||||
|
info!("Starting gRPC server on {}", self.address);
|
||||||
|
|
||||||
|
let service = EmotionServiceImpl {
|
||||||
|
us_db: self.us_db.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
tonic::transport::Server::builder()
|
||||||
|
.add_service(EmotionServiceServer::new(service))
|
||||||
|
.serve(addr)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct EmotionServiceImpl {
|
||||||
|
us_db: Arc<Mutex<Connection>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tonic::async_trait]
|
||||||
|
impl EmotionServiceTrait for EmotionServiceImpl {
|
||||||
|
async fn get_emotion(
|
||||||
|
&self,
|
||||||
|
request: Request<GetEmotionRequest>,
|
||||||
|
) -> Result<Response<Emotion>, Status> {
|
||||||
|
let req = request.into_inner();
|
||||||
|
let conn = self.us_db.lock().await;
|
||||||
|
|
||||||
|
match db::get_emotion_by_message_id(&conn, req.message_id) {
|
||||||
|
Ok(Some(emotion)) => Ok(Response::new(emotion_to_proto(emotion))),
|
||||||
|
Ok(None) => Err(Status::not_found(format!(
|
||||||
|
"Emotion not found for message_id: {}",
|
||||||
|
req.message_id
|
||||||
|
))),
|
||||||
|
Err(e) => {
|
||||||
|
error!("Database error: {}", e);
|
||||||
|
Err(Status::internal("Database error"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_emotions(
|
||||||
|
&self,
|
||||||
|
request: Request<GetEmotionsRequest>,
|
||||||
|
) -> Result<Response<EmotionsResponse>, Status> {
|
||||||
|
let req = request.into_inner();
|
||||||
|
let conn = self.us_db.lock().await;
|
||||||
|
|
||||||
|
let emotion_filter = req.emotion_filter.as_deref();
|
||||||
|
let min_confidence = req.min_confidence;
|
||||||
|
let limit = req.limit.map(|l| l as i32);
|
||||||
|
let offset = req.offset.map(|o| o as i32);
|
||||||
|
|
||||||
|
match db::list_emotions(&conn, emotion_filter, min_confidence, limit, offset) {
|
||||||
|
Ok(emotions) => {
|
||||||
|
let total_count = db::count_emotions(&conn).unwrap_or(0);
|
||||||
|
Ok(Response::new(EmotionsResponse {
|
||||||
|
emotions: emotions.into_iter().map(emotion_to_proto).collect(),
|
||||||
|
total_count,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Database error: {}", e);
|
||||||
|
Err(Status::internal("Database error"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_all_emotions(
|
||||||
|
&self,
|
||||||
|
_request: Request<Empty>,
|
||||||
|
) -> Result<Response<EmotionsResponse>, Status> {
|
||||||
|
let conn = self.us_db.lock().await;
|
||||||
|
|
||||||
|
match db::list_emotions(&conn, None, None, None, None) {
|
||||||
|
Ok(emotions) => {
|
||||||
|
let total_count = emotions.len() as i32;
|
||||||
|
Ok(Response::new(EmotionsResponse {
|
||||||
|
emotions: emotions.into_iter().map(emotion_to_proto).collect(),
|
||||||
|
total_count,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Database error: {}", e);
|
||||||
|
Err(Status::internal("Database error"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_emotion(
|
||||||
|
&self,
|
||||||
|
request: Request<UpdateEmotionRequest>,
|
||||||
|
) -> Result<Response<EmotionResponse>, Status> {
|
||||||
|
let req = request.into_inner();
|
||||||
|
let conn = self.us_db.lock().await;
|
||||||
|
|
||||||
|
match db::update_emotion(&conn, req.message_id, &req.emotion, req.confidence) {
|
||||||
|
Ok(_) => {
|
||||||
|
// If notes are provided, add to training set
|
||||||
|
if let Some(notes) = req.notes {
|
||||||
|
if let Ok(Some(msg)) = db::get_message(&conn, req.message_id) {
|
||||||
|
if let Some(text) = msg.text {
|
||||||
|
let _ = db::insert_training_sample(
|
||||||
|
&conn,
|
||||||
|
Some(req.message_id),
|
||||||
|
&text,
|
||||||
|
&req.emotion,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetch the updated emotion
|
||||||
|
match db::get_emotion_by_message_id(&conn, req.message_id) {
|
||||||
|
Ok(Some(emotion)) => Ok(Response::new(EmotionResponse {
|
||||||
|
success: true,
|
||||||
|
message: "Emotion updated successfully".to_string(),
|
||||||
|
emotion: Some(emotion_to_proto(emotion)),
|
||||||
|
})),
|
||||||
|
_ => Ok(Response::new(EmotionResponse {
|
||||||
|
success: true,
|
||||||
|
message: "Emotion updated successfully".to_string(),
|
||||||
|
emotion: None,
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Database error: {}", e);
|
||||||
|
Err(Status::internal("Database error"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn batch_update_emotions(
|
||||||
|
&self,
|
||||||
|
request: Request<tonic::Streaming<UpdateEmotionRequest>>,
|
||||||
|
) -> Result<Response<EmotionResponse>, Status> {
|
||||||
|
let mut stream = request.into_inner();
|
||||||
|
let mut count = 0;
|
||||||
|
|
||||||
|
while let Some(req) = stream.message().await? {
|
||||||
|
let conn = self.us_db.lock().await;
|
||||||
|
match db::update_emotion(&conn, req.message_id, &req.emotion, req.confidence) {
|
||||||
|
Ok(_) => {
|
||||||
|
count += 1;
|
||||||
|
if let Some(notes) = req.notes {
|
||||||
|
if let Ok(Some(msg)) = db::get_message(&conn, req.message_id) {
|
||||||
|
if let Some(text) = msg.text {
|
||||||
|
let _ = db::insert_training_sample(
|
||||||
|
&conn,
|
||||||
|
Some(req.message_id),
|
||||||
|
&text,
|
||||||
|
&req.emotion,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to update emotion for message {}: {}", req.message_id, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(conn);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Response::new(EmotionResponse {
|
||||||
|
success: true,
|
||||||
|
message: format!("Updated {} emotions", count),
|
||||||
|
emotion: None,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_emotion(
|
||||||
|
&self,
|
||||||
|
request: Request<DeleteEmotionRequest>,
|
||||||
|
) -> Result<Response<EmotionResponse>, Status> {
|
||||||
|
let req = request.into_inner();
|
||||||
|
let conn = self.us_db.lock().await;
|
||||||
|
|
||||||
|
match db::delete_emotion(&conn, req.id) {
|
||||||
|
Ok(_) => Ok(Response::new(EmotionResponse {
|
||||||
|
success: true,
|
||||||
|
message: format!("Emotion {} deleted successfully", req.id),
|
||||||
|
emotion: None,
|
||||||
|
})),
|
||||||
|
Err(e) => {
|
||||||
|
error!("Database error: {}", e);
|
||||||
|
Err(Status::internal("Database error"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emotion_to_proto(emotion: crate::models::Emotion) -> Emotion {
|
||||||
|
Emotion {
|
||||||
|
id: emotion.id,
|
||||||
|
message_id: emotion.message_id,
|
||||||
|
emotion: emotion.emotion,
|
||||||
|
confidence: emotion.confidence,
|
||||||
|
model_version: emotion.model_version,
|
||||||
|
created_at: emotion.created_at.timestamp(),
|
||||||
|
updated_at: emotion.updated_at.timestamp(),
|
||||||
|
}
|
||||||
|
}
|
||||||
7
crates/server/src/services/mod.rs
Normal file
7
crates/server/src/services/mod.rs
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
pub mod chat_poller;
|
||||||
|
pub mod embedding_service;
|
||||||
|
pub mod emotion_service;
|
||||||
|
|
||||||
|
pub use chat_poller::ChatPollerService;
|
||||||
|
pub use embedding_service::EmbeddingService;
|
||||||
|
pub use emotion_service::EmotionService;
|
||||||
114
crates/server/src/sync_plugin.rs
Normal file
114
crates/server/src/sync_plugin.rs
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
use bevy::prelude::*;
|
||||||
|
use lib::sync::{Syncable, SyncMessage};
|
||||||
|
use crate::components::*;
|
||||||
|
|
||||||
|
/// Bevy plugin for transparent CRDT sync via gossip
|
||||||
|
pub struct SyncPlugin;
|
||||||
|
|
||||||
|
impl Plugin for SyncPlugin {
|
||||||
|
fn build(&self, app: &mut App) {
|
||||||
|
app.add_systems(Update, (
|
||||||
|
publish_sync_ops,
|
||||||
|
receive_sync_ops,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trait for Bevy resources that can be synced
|
||||||
|
pub trait SyncedResource: Resource + Syncable + Clone + Send + Sync + 'static {}
|
||||||
|
|
||||||
|
/// Queue of sync operations to publish
|
||||||
|
#[derive(Resource, Default)]
|
||||||
|
pub struct SyncOpQueue<T: Syncable> {
|
||||||
|
pub ops: Vec<T::Operation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Syncable> SyncOpQueue<T> {
|
||||||
|
pub fn push(&mut self, op: T::Operation) {
|
||||||
|
self.ops.push(op);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// System to publish sync operations to gossip
|
||||||
|
fn publish_sync_ops<T: SyncedResource>(
|
||||||
|
mut queue: ResMut<SyncOpQueue<T>>,
|
||||||
|
resource: Res<T>,
|
||||||
|
sender: Option<Res<IrohGossipSender>>,
|
||||||
|
) {
|
||||||
|
if sender.is_none() || queue.ops.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sender = sender.unwrap();
|
||||||
|
let sender_guard = sender.sender.lock();
|
||||||
|
|
||||||
|
for op in queue.ops.drain(..) {
|
||||||
|
let sync_msg = resource.create_sync_message(op);
|
||||||
|
|
||||||
|
match sync_msg.to_bytes() {
|
||||||
|
Ok(bytes) => {
|
||||||
|
println!("Publishing sync operation: {} bytes", bytes.len());
|
||||||
|
// TODO: Actually send via gossip
|
||||||
|
// sender_guard.broadcast(bytes)?;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("Failed to serialize sync operation: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// System to receive and apply sync operations from gossip
|
||||||
|
fn receive_sync_ops<T: SyncedResource>(
|
||||||
|
mut resource: ResMut<T>,
|
||||||
|
receiver: Option<Res<IrohGossipReceiver>>,
|
||||||
|
) {
|
||||||
|
if receiver.is_none() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Poll receiver for messages
|
||||||
|
// For each message:
|
||||||
|
// 1. Deserialize SyncMessage<T::Operation>
|
||||||
|
// 2. Apply to resource with resource.apply_sync_op(&op)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper to register a synced resource
|
||||||
|
pub trait SyncedResourceExt {
|
||||||
|
fn add_synced_resource<T: SyncedResource>(&mut self) -> &mut Self;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SyncedResourceExt for App {
|
||||||
|
fn add_synced_resource<T: SyncedResource>(&mut self) -> &mut Self {
|
||||||
|
self.init_resource::<SyncOpQueue<T>>();
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Example synced resource
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use lib::sync::synced;
|
||||||
|
|
||||||
|
#[synced]
|
||||||
|
pub struct TestConfig {
|
||||||
|
pub value: i32,
|
||||||
|
|
||||||
|
#[sync(skip)]
|
||||||
|
node_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Resource for TestConfig {}
|
||||||
|
impl SyncedResource for TestConfig {}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sync_plugin() {
|
||||||
|
let mut app = App::new();
|
||||||
|
app.add_plugins(MinimalPlugins);
|
||||||
|
app.add_plugins(SyncPlugin);
|
||||||
|
app.add_synced_resource::<TestConfig>();
|
||||||
|
|
||||||
|
// TODO: Test that operations are queued and published
|
||||||
|
}
|
||||||
|
}
|
||||||
12
crates/server/src/systems/database.rs
Normal file
12
crates/server/src/systems/database.rs
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
use bevy::prelude::*;
|
||||||
|
|
||||||
|
use crate::components::*;
|
||||||
|
|
||||||
|
/// System: Poll chat.db for new messages using Bevy's task system
|
||||||
|
pub fn poll_chat_db(
|
||||||
|
_config: Res<AppConfig>,
|
||||||
|
_db: Res<Database>,
|
||||||
|
) {
|
||||||
|
// TODO: Use Bevy's AsyncComputeTaskPool to poll chat.db
|
||||||
|
// This will replace the tokio::spawn chat poller
|
||||||
|
}
|
||||||
116
crates/server/src/systems/gossip.rs
Normal file
116
crates/server/src/systems/gossip.rs
Normal file
@@ -0,0 +1,116 @@
|
|||||||
|
use bevy::prelude::*;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::components::*;
|
||||||
|
|
||||||
|
/// System: Poll the gossip init task and insert resources when complete
|
||||||
|
pub fn poll_gossip_init(
|
||||||
|
mut commands: Commands,
|
||||||
|
mut init_task: Option<ResMut<GossipInitTask>>,
|
||||||
|
) {
|
||||||
|
if let Some(mut task) = init_task {
|
||||||
|
// Check if the task is finished (non-blocking)
|
||||||
|
if let Some(result) = bevy::tasks::block_on(bevy::tasks::futures_lite::future::poll_once(&mut task.0)) {
|
||||||
|
if let Some((endpoint, gossip, router, sender, receiver)) = result {
|
||||||
|
println!("Inserting gossip resources");
|
||||||
|
|
||||||
|
// Insert all the resources
|
||||||
|
commands.insert_resource(IrohEndpoint {
|
||||||
|
endpoint,
|
||||||
|
node_id: "TODO".to_string(), // TODO: Figure out how to get node_id in iroh 0.95
|
||||||
|
});
|
||||||
|
commands.insert_resource(IrohGossipHandle { gossip });
|
||||||
|
commands.insert_resource(IrohRouter { router });
|
||||||
|
commands.insert_resource(IrohGossipSender {
|
||||||
|
sender: Arc::new(Mutex::new(sender)),
|
||||||
|
});
|
||||||
|
commands.insert_resource(IrohGossipReceiver {
|
||||||
|
receiver: Arc::new(Mutex::new(receiver)),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Remove the init task
|
||||||
|
commands.remove_resource::<GossipInitTask>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// System: Detect new messages in SQLite that need to be published to gossip
|
||||||
|
pub fn detect_new_messages(
|
||||||
|
_db: Res<Database>,
|
||||||
|
_last_synced: Local<i64>,
|
||||||
|
_publish_events: MessageWriter<PublishMessageEvent>,
|
||||||
|
) {
|
||||||
|
// TODO: Query SQLite for messages with rowid > last_synced
|
||||||
|
// When we detect new messages, we'll send PublishMessageEvent
|
||||||
|
}
|
||||||
|
|
||||||
|
/// System: Publish messages to gossip when PublishMessageEvent is triggered
|
||||||
|
pub fn publish_to_gossip(
|
||||||
|
mut events: MessageReader<PublishMessageEvent>,
|
||||||
|
sender: Option<Res<IrohGossipSender>>,
|
||||||
|
endpoint: Option<Res<IrohEndpoint>>,
|
||||||
|
) {
|
||||||
|
if sender.is_none() || endpoint.is_none() {
|
||||||
|
// Gossip not initialized yet, skip
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sender = sender.unwrap();
|
||||||
|
let endpoint = endpoint.unwrap();
|
||||||
|
|
||||||
|
for event in events.read() {
|
||||||
|
println!("Publishing message {} to gossip", event.message.rowid);
|
||||||
|
|
||||||
|
// Create sync message
|
||||||
|
let sync_message = SyncMessage {
|
||||||
|
message: event.message.clone(),
|
||||||
|
sync_timestamp: chrono::Utc::now().timestamp(),
|
||||||
|
publisher_node_id: endpoint.node_id.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Serialize the message
|
||||||
|
match serialize_sync_message(&sync_message) {
|
||||||
|
Ok(bytes) => {
|
||||||
|
// TODO: Publish to gossip
|
||||||
|
// For now, just log that we would publish
|
||||||
|
println!("Would publish {} bytes to gossip", bytes.len());
|
||||||
|
|
||||||
|
// Note: Direct async broadcasting from Bevy systems is tricky due to Sync requirements
|
||||||
|
// We'll need to use a different approach, possibly with channels or a dedicated task
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("Failed to serialize sync message: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// System: Receive messages from gossip
|
||||||
|
pub fn receive_from_gossip(
|
||||||
|
mut _gossip_events: MessageWriter<GossipMessageReceived>,
|
||||||
|
receiver: Option<Res<IrohGossipReceiver>>,
|
||||||
|
) {
|
||||||
|
if receiver.is_none() {
|
||||||
|
// Gossip not initialized yet, skip
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Implement proper async message reception
|
||||||
|
// This will require spawning a long-running task that listens for gossip events
|
||||||
|
// and sends them as Bevy messages. For now, this is a placeholder.
|
||||||
|
}
|
||||||
|
|
||||||
|
/// System: Save received gossip messages to SQLite
|
||||||
|
pub fn save_gossip_messages(
|
||||||
|
mut events: MessageReader<GossipMessageReceived>,
|
||||||
|
_db: Res<Database>,
|
||||||
|
) {
|
||||||
|
for event in events.read() {
|
||||||
|
println!("Received message {} from gossip (published by {})",
|
||||||
|
event.sync_message.message.rowid,
|
||||||
|
event.sync_message.publisher_node_id);
|
||||||
|
// TODO: Save to SQLite if we don't already have it
|
||||||
|
}
|
||||||
|
}
|
||||||
7
crates/server/src/systems/mod.rs
Normal file
7
crates/server/src/systems/mod.rs
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
pub mod database;
|
||||||
|
pub mod gossip;
|
||||||
|
pub mod setup;
|
||||||
|
|
||||||
|
pub use database::*;
|
||||||
|
pub use gossip::*;
|
||||||
|
pub use setup::*;
|
||||||
22
crates/server/src/systems/setup.rs
Normal file
22
crates/server/src/systems/setup.rs
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
use bevy::prelude::*;
|
||||||
|
use bevy::tasks::AsyncComputeTaskPool;
|
||||||
|
|
||||||
|
use crate::components::*;
|
||||||
|
|
||||||
|
/// Startup system: Initialize database
|
||||||
|
pub fn setup_database(_db: Res<Database>) {
|
||||||
|
println!("Database resource initialized");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Startup system: Initialize Iroh gossip
|
||||||
|
pub fn setup_gossip(mut commands: Commands, topic: Res<GossipTopic>) {
|
||||||
|
println!("Setting up Iroh gossip for topic: {:?}", topic.0);
|
||||||
|
|
||||||
|
let topic_id = topic.0;
|
||||||
|
|
||||||
|
// TODO: Initialize gossip properly
|
||||||
|
// For now, skip async initialization due to Sync requirements in Bevy tasks
|
||||||
|
// We'll need to use a different initialization strategy
|
||||||
|
|
||||||
|
println!("Gossip initialization skipped (TODO: implement proper async init)");
|
||||||
|
}
|
||||||
15
crates/sync-macros/Cargo.toml
Normal file
15
crates/sync-macros/Cargo.toml
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
[package]
|
||||||
|
name = "sync-macros"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
proc-macro = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
syn = { version = "2.0", features = ["full"] }
|
||||||
|
quote = "1.0"
|
||||||
|
proc-macro2 = "1.0"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
lib = { path = "../lib" }
|
||||||
345
crates/sync-macros/src/lib.rs
Normal file
345
crates/sync-macros/src/lib.rs
Normal file
@@ -0,0 +1,345 @@
|
|||||||
|
use proc_macro::TokenStream;
|
||||||
|
use quote::{quote, format_ident};
|
||||||
|
use syn::{parse_macro_input, DeriveInput, Data, Fields, Type, ItemStruct};
|
||||||
|
|
||||||
|
/// Attribute macro for transparent CRDT sync
|
||||||
|
///
|
||||||
|
/// Transforms your struct to use CRDTs internally while keeping the API simple.
|
||||||
|
///
|
||||||
|
/// # Example
|
||||||
|
/// ```
|
||||||
|
/// #[synced]
|
||||||
|
/// struct EmotionGradientConfig {
|
||||||
|
/// canvas_width: f32, // Becomes SyncedValue<f32> internally
|
||||||
|
/// canvas_height: f32, // Auto-generates getters/setters
|
||||||
|
///
|
||||||
|
/// #[sync(skip)]
|
||||||
|
/// node_id: String, // Not synced
|
||||||
|
/// }
|
||||||
|
///
|
||||||
|
/// // Use it like a normal struct:
|
||||||
|
/// let mut config = EmotionGradientConfig::new("node1".into());
|
||||||
|
/// config.set_canvas_width(1024.0); // Auto-generates sync operation
|
||||||
|
/// println!("Width: {}", config.canvas_width()); // Transparent access
|
||||||
|
/// ```
|
||||||
|
#[proc_macro_attribute]
|
||||||
|
pub fn synced(_attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||||
|
let input = parse_macro_input!(item as ItemStruct);
|
||||||
|
let name = &input.ident;
|
||||||
|
let vis = &input.vis;
|
||||||
|
let op_enum_name = format_ident!("{}Op", name);
|
||||||
|
|
||||||
|
let fields = match &input.fields {
|
||||||
|
Fields::Named(fields) => &fields.named,
|
||||||
|
_ => panic!("synced only supports structs with named fields"),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut internal_fields = Vec::new();
|
||||||
|
let mut field_getters = Vec::new();
|
||||||
|
let mut field_setters = Vec::new();
|
||||||
|
let mut op_variants = Vec::new();
|
||||||
|
let mut apply_arms = Vec::new();
|
||||||
|
let mut merge_code = Vec::new();
|
||||||
|
let mut new_params = Vec::new();
|
||||||
|
let mut new_init = Vec::new();
|
||||||
|
|
||||||
|
for field in fields {
|
||||||
|
let field_name = field.ident.as_ref().unwrap();
|
||||||
|
let field_vis = &field.vis;
|
||||||
|
let field_type = &field.ty;
|
||||||
|
|
||||||
|
// Check if field should be skipped
|
||||||
|
let should_skip = field.attrs.iter().any(|attr| {
|
||||||
|
attr.path().is_ident("sync")
|
||||||
|
&& attr
|
||||||
|
.parse_args::<syn::Ident>()
|
||||||
|
.map(|i| i == "skip")
|
||||||
|
.unwrap_or(false)
|
||||||
|
});
|
||||||
|
|
||||||
|
if should_skip {
|
||||||
|
// Keep as-is, no wrapping
|
||||||
|
internal_fields.push(quote! {
|
||||||
|
#field_vis #field_name: #field_type
|
||||||
|
});
|
||||||
|
new_params.push(quote! { #field_name: #field_type });
|
||||||
|
new_init.push(quote! { #field_name });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wrap in SyncedValue
|
||||||
|
internal_fields.push(quote! {
|
||||||
|
#field_name: lib::sync::SyncedValue<#field_type>
|
||||||
|
});
|
||||||
|
|
||||||
|
// Generate getter
|
||||||
|
field_getters.push(quote! {
|
||||||
|
#field_vis fn #field_name(&self) -> &#field_type {
|
||||||
|
self.#field_name.get()
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Generate setter that returns operation
|
||||||
|
let setter_name = format_ident!("set_{}", field_name);
|
||||||
|
let op_variant = format_ident!(
|
||||||
|
"Set{}",
|
||||||
|
field_name
|
||||||
|
.to_string()
|
||||||
|
.chars()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, c)| if i == 0 {
|
||||||
|
c.to_ascii_uppercase()
|
||||||
|
} else {
|
||||||
|
c
|
||||||
|
})
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
|
||||||
|
field_setters.push(quote! {
|
||||||
|
#field_vis fn #setter_name(&mut self, value: #field_type) -> #op_enum_name {
|
||||||
|
let op = #op_enum_name::#op_variant {
|
||||||
|
value: value.clone(),
|
||||||
|
timestamp: chrono::Utc::now(),
|
||||||
|
node_id: self.node_id().clone(),
|
||||||
|
};
|
||||||
|
self.#field_name.set(value, self.node_id().clone());
|
||||||
|
op
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Generate operation variant
|
||||||
|
op_variants.push(quote! {
|
||||||
|
#op_variant {
|
||||||
|
value: #field_type,
|
||||||
|
timestamp: chrono::DateTime<chrono::Utc>,
|
||||||
|
node_id: String,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Generate apply arm
|
||||||
|
apply_arms.push(quote! {
|
||||||
|
#op_enum_name::#op_variant { value, timestamp, node_id } => {
|
||||||
|
self.#field_name.apply_lww(value.clone(), timestamp.clone(), node_id.clone());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Generate merge code
|
||||||
|
merge_code.push(quote! {
|
||||||
|
self.#field_name.merge(&other.#field_name);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Add to new() parameters
|
||||||
|
new_params.push(quote! { #field_name: #field_type });
|
||||||
|
new_init.push(quote! {
|
||||||
|
#field_name: lib::sync::SyncedValue::new(#field_name, node_id.clone())
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let expanded = quote! {
|
||||||
|
/// Sync operations enum
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(tag = "type")]
|
||||||
|
#vis enum #op_enum_name {
|
||||||
|
#(#op_variants),*
|
||||||
|
}
|
||||||
|
|
||||||
|
impl #op_enum_name {
|
||||||
|
pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
|
||||||
|
Ok(serde_json::to_vec(self)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self> {
|
||||||
|
Ok(serde_json::from_slice(bytes)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
#vis struct #name {
|
||||||
|
#(#internal_fields),*
|
||||||
|
}
|
||||||
|
|
||||||
|
impl #name {
|
||||||
|
#vis fn new(#(#new_params),*) -> Self {
|
||||||
|
Self {
|
||||||
|
#(#new_init),*
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Transparent field accessors
|
||||||
|
#(#field_getters)*
|
||||||
|
|
||||||
|
/// Field setters that generate sync operations
|
||||||
|
#(#field_setters)*
|
||||||
|
|
||||||
|
/// Apply a sync operation from another node
|
||||||
|
#vis fn apply_op(&mut self, op: &#op_enum_name) {
|
||||||
|
match op {
|
||||||
|
#(#apply_arms),*
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Merge state from another instance
|
||||||
|
#vis fn merge(&mut self, other: &Self) {
|
||||||
|
#(#merge_code)*
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl lib::sync::Syncable for #name {
|
||||||
|
type Operation = #op_enum_name;
|
||||||
|
|
||||||
|
fn apply_sync_op(&mut self, op: &Self::Operation) {
|
||||||
|
self.apply_op(op);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn node_id(&self) -> &lib::sync::NodeId {
|
||||||
|
// Assume there's a node_id field marked with #[sync(skip)]
|
||||||
|
&self.node_id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
TokenStream::from(expanded)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Old derive macro - kept for backwards compatibility
|
||||||
|
#[proc_macro_derive(Synced, attributes(sync))]
|
||||||
|
pub fn derive_synced(input: TokenStream) -> TokenStream {
|
||||||
|
let input = parse_macro_input!(input as DeriveInput);
|
||||||
|
let name = &input.ident;
|
||||||
|
let op_enum_name = format_ident!("{}Op", name);
|
||||||
|
|
||||||
|
let fields = match &input.data {
|
||||||
|
Data::Struct(data) => match &data.fields {
|
||||||
|
Fields::Named(fields) => &fields.named,
|
||||||
|
_ => panic!("Synced only supports structs with named fields"),
|
||||||
|
},
|
||||||
|
_ => panic!("Synced only supports structs"),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut field_ops = Vec::new();
|
||||||
|
let mut apply_arms = Vec::new();
|
||||||
|
let mut setter_methods = Vec::new();
|
||||||
|
let mut merge_code = Vec::new();
|
||||||
|
|
||||||
|
for field in fields {
|
||||||
|
let field_name = field.ident.as_ref().unwrap();
|
||||||
|
let field_type = &field.ty;
|
||||||
|
|
||||||
|
// Check if field should be skipped
|
||||||
|
let should_skip = field.attrs.iter()
|
||||||
|
.any(|attr| {
|
||||||
|
attr.path().is_ident("sync") &&
|
||||||
|
attr.parse_args::<syn::Ident>()
|
||||||
|
.map(|i| i == "skip")
|
||||||
|
.unwrap_or(false)
|
||||||
|
});
|
||||||
|
|
||||||
|
if should_skip {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let op_variant = format_ident!("Set{}",
|
||||||
|
field_name.to_string()
|
||||||
|
.chars()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, c)| if i == 0 { c.to_ascii_uppercase() } else { c })
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
|
||||||
|
let setter_name = format_ident!("set_{}", field_name);
|
||||||
|
|
||||||
|
// Determine CRDT strategy based on type
|
||||||
|
let crdt_strategy = get_crdt_strategy(field_type);
|
||||||
|
|
||||||
|
match crdt_strategy.as_str() {
|
||||||
|
"lww" => {
|
||||||
|
// LWW for simple types
|
||||||
|
field_ops.push(quote! {
|
||||||
|
#op_variant {
|
||||||
|
value: #field_type,
|
||||||
|
timestamp: chrono::DateTime<chrono::Utc>,
|
||||||
|
node_id: String,
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
apply_arms.push(quote! {
|
||||||
|
#op_enum_name::#op_variant { value, timestamp, node_id } => {
|
||||||
|
self.#field_name.apply_lww(value.clone(), timestamp.clone(), node_id.clone());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
setter_methods.push(quote! {
|
||||||
|
pub fn #setter_name(&mut self, value: #field_type) -> #op_enum_name {
|
||||||
|
let op = #op_enum_name::#op_variant {
|
||||||
|
value: value.clone(),
|
||||||
|
timestamp: chrono::Utc::now(),
|
||||||
|
node_id: self.node_id().clone(),
|
||||||
|
};
|
||||||
|
self.#field_name = lib::sync::SyncedValue::new(value, self.node_id().clone());
|
||||||
|
op
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
merge_code.push(quote! {
|
||||||
|
self.#field_name.merge(&other.#field_name);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
// Default to LWW
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let expanded = quote! {
|
||||||
|
/// Auto-generated sync operations enum
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(tag = "type")]
|
||||||
|
pub enum #op_enum_name {
|
||||||
|
#(#field_ops),*
|
||||||
|
}
|
||||||
|
|
||||||
|
impl #op_enum_name {
|
||||||
|
pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
|
||||||
|
Ok(serde_json::to_vec(self)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self> {
|
||||||
|
Ok(serde_json::from_slice(bytes)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl #name {
|
||||||
|
/// Apply a sync operation from another node
|
||||||
|
pub fn apply_op(&mut self, op: &#op_enum_name) {
|
||||||
|
match op {
|
||||||
|
#(#apply_arms),*
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Merge state from another instance
|
||||||
|
pub fn merge(&mut self, other: &Self) {
|
||||||
|
#(#merge_code)*
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Auto-generated setter methods that create sync ops
|
||||||
|
#(#setter_methods)*
|
||||||
|
}
|
||||||
|
|
||||||
|
impl lib::sync::Syncable for #name {
|
||||||
|
type Operation = #op_enum_name;
|
||||||
|
|
||||||
|
fn apply_sync_op(&mut self, op: &Self::Operation) {
|
||||||
|
self.apply_op(op);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
TokenStream::from(expanded)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Determine CRDT strategy based on field type
|
||||||
|
fn get_crdt_strategy(_ty: &Type) -> String {
|
||||||
|
// For now, default everything to LWW
|
||||||
|
// TODO: Detect HashMap -> use Map, Vec -> use ORSet, etc.
|
||||||
|
"lww".to_string()
|
||||||
|
}
|
||||||
1653
docs/rfcs/0001-crdt-gossip-sync.md
Normal file
1653
docs/rfcs/0001-crdt-gossip-sync.md
Normal file
File diff suppressed because it is too large
Load Diff
566
docs/rfcs/0002-persistence-strategy.md
Normal file
566
docs/rfcs/0002-persistence-strategy.md
Normal file
@@ -0,0 +1,566 @@
|
|||||||
|
# RFC 0002: Persistence Strategy for Battery-Efficient State Management
|
||||||
|
|
||||||
|
**Status:** Draft
|
||||||
|
**Authors:** Sienna
|
||||||
|
**Created:** 2025-11-15
|
||||||
|
**Related:** RFC 0001 (CRDT Sync Protocol)
|
||||||
|
|
||||||
|
## Abstract
|
||||||
|
|
||||||
|
This RFC defines a persistence strategy that balances data durability with battery efficiency for mobile platforms (iPad). The core challenge: Bevy runs at 60fps and generates continuous state changes, but we can't write to SQLite on every frame without destroying battery life and flash storage.
|
||||||
|
|
||||||
|
## The Problem
|
||||||
|
|
||||||
|
**Naive approach (bad)**:
|
||||||
|
```rust
|
||||||
|
fn sync_to_db_system(query: Query<&NetworkedEntity, Changed<Transform>>) {
|
||||||
|
for entity in query.iter() {
|
||||||
|
db.execute("UPDATE components SET data = ? WHERE entity_id = ?", ...)?;
|
||||||
|
// This runs 60 times per second!
|
||||||
|
// iPad battery: 💀
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Why this is terrible**:
|
||||||
|
- SQLite writes trigger `fsync()` syscalls (flush to physical storage)
|
||||||
|
- Each `fsync()` on iOS can take 5-20ms and drains battery significantly
|
||||||
|
- At 60fps with multiple entities, we'd be doing hundreds of disk writes per second
|
||||||
|
- Flash wear: mobile devices have limited write cycles
|
||||||
|
- User moves object around → hundreds of unnecessary writes of intermediate positions
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
1. **Survive crashes**: If the app crashes, user shouldn't lose more than a few seconds of work
|
||||||
|
2. **Battery efficient**: Minimize disk I/O, especially `fsync()` calls
|
||||||
|
3. **Flash-friendly**: Reduce write amplification on mobile storage
|
||||||
|
4. **Low latency**: Persistence shouldn't block rendering or input
|
||||||
|
5. **Recoverable**: On startup, we should be able to reconstruct recent state
|
||||||
|
|
||||||
|
## Categorizing Data by Persistence Needs
|
||||||
|
|
||||||
|
Not all data is equal. We need to categorize by how critical immediate persistence is:
|
||||||
|
|
||||||
|
### Tier 1: Critical State (Persist Immediately)
|
||||||
|
|
||||||
|
**What**: State that's hard or impossible to reconstruct if lost
|
||||||
|
- User-created entities (the fact that they exist)
|
||||||
|
- Operation log entries (for CRDT sync)
|
||||||
|
- Vector clock state (for causality tracking)
|
||||||
|
- Document metadata (name, creation time, etc.)
|
||||||
|
|
||||||
|
**Why**: These are the "source of truth" - if we lose them, data is gone
|
||||||
|
|
||||||
|
**Strategy**: Write to database within ~1 second of creation, but still batched
|
||||||
|
|
||||||
|
### Tier 2: Derived State (Defer and Batch)
|
||||||
|
|
||||||
|
**What**: State that can be reconstructed or is constantly changing
|
||||||
|
- Entity positions during drag operations
|
||||||
|
- Transform components (position, rotation, scale)
|
||||||
|
- UI state (selected items, viewport position)
|
||||||
|
- Temporary drawing strokes in progress
|
||||||
|
|
||||||
|
**Why**: These change rapidly and the intermediate states aren't valuable
|
||||||
|
|
||||||
|
**Strategy**: Batch writes, flush every 5-10 seconds or on specific events
|
||||||
|
|
||||||
|
### Tier 3: Ephemeral State (Never Persist)
|
||||||
|
|
||||||
|
**What**: State that only matters during current session
|
||||||
|
- Remote peer cursors
|
||||||
|
- Presence indicators (who's online)
|
||||||
|
- Network connection status
|
||||||
|
- Frame-rate metrics
|
||||||
|
|
||||||
|
**Why**: These are meaningless after restart
|
||||||
|
|
||||||
|
**Strategy**: Keep in-memory only (Bevy resources, not components)
|
||||||
|
|
||||||
|
## Write Strategy: The Three-Buffer System
|
||||||
|
|
||||||
|
We use a three-tier approach to minimize disk writes while maintaining durability:
|
||||||
|
|
||||||
|
### Layer 1: In-Memory Dirty Tracking (0ms latency)
|
||||||
|
|
||||||
|
Bevy change detection marks components as dirty, but we don't write immediately. Instead, we maintain a dirty set:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[derive(Resource)]
|
||||||
|
struct DirtyEntities {
|
||||||
|
// Entities with changes not yet in write buffer
|
||||||
|
entities: HashSet<Uuid>,
|
||||||
|
components: HashMap<Uuid, HashSet<String>>, // entity → dirty component types
|
||||||
|
last_modified: HashMap<Uuid, Instant>, // when was it last changed
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Update frequency**: Every frame (cheap - just memory operations)
|
||||||
|
|
||||||
|
### Layer 2: Write Buffer (100ms-1s batching)
|
||||||
|
|
||||||
|
Periodically (every 100ms-1s), we collect dirty entities and prepare a write batch:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[derive(Resource)]
|
||||||
|
struct WriteBuffer {
|
||||||
|
// Pending writes not yet committed to SQLite
|
||||||
|
pending_operations: Vec<PersistenceOp>,
|
||||||
|
last_flush: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum PersistenceOp {
|
||||||
|
UpsertEntity { id: Uuid, data: EntityData },
|
||||||
|
UpsertComponent { entity_id: Uuid, component_type: String, data: Vec<u8> },
|
||||||
|
LogOperation { node_id: NodeId, seq: u64, op: Vec<u8> },
|
||||||
|
UpdateVectorClock { node_id: NodeId, counter: u64 },
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Update frequency**: Every 100ms-1s (configurable based on battery level)
|
||||||
|
|
||||||
|
**Strategy**: Accumulate operations in memory, then batch-write them
|
||||||
|
|
||||||
|
### Layer 3: SQLite with WAL Mode (5-10s commit interval)
|
||||||
|
|
||||||
|
Write buffer is flushed to SQLite, but we don't call `fsync()` immediately. Instead, we use WAL mode and control checkpoint timing:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Enable Write-Ahead Logging
|
||||||
|
PRAGMA journal_mode = WAL;
|
||||||
|
|
||||||
|
-- Don't auto-checkpoint on every transaction
|
||||||
|
PRAGMA wal_autocheckpoint = 0;
|
||||||
|
|
||||||
|
-- Synchronous = NORMAL (fsync WAL on commit, but not every write)
|
||||||
|
PRAGMA synchronous = NORMAL;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Update frequency**: Manual checkpoints every 5-10 seconds (or on specific events)
|
||||||
|
|
||||||
|
## Flush Events: When to Force Persistence
|
||||||
|
|
||||||
|
Certain events require immediate persistence (within 1 second):
|
||||||
|
|
||||||
|
### 1. Entity Creation
|
||||||
|
When user creates a new entity, we need to persist its existence quickly:
|
||||||
|
- Add to write buffer immediately
|
||||||
|
- Trigger flush within 1 second
|
||||||
|
|
||||||
|
### 2. Major User Actions
|
||||||
|
Actions that represent "savepoints" in user's mental model:
|
||||||
|
- Finishing a drawing stroke (stroke start → immediate, intermediate points → batched, stroke end → flush)
|
||||||
|
- Deleting entities
|
||||||
|
- Changing document metadata
|
||||||
|
- Undo/redo operations
|
||||||
|
|
||||||
|
### 3. Application State Transitions
|
||||||
|
State changes that might precede app termination:
|
||||||
|
- App going to background (iOS `applicationWillResignActive`)
|
||||||
|
- Low memory warning
|
||||||
|
- User explicitly saving (if we have a save button)
|
||||||
|
- Switching documents/workspaces
|
||||||
|
|
||||||
|
### 4. Network Events
|
||||||
|
Sync protocol events that need persistence:
|
||||||
|
- Receiving operation log entries from peers
|
||||||
|
- Vector clock updates (every 5 operations or 5 seconds, whichever comes first)
|
||||||
|
|
||||||
|
### 5. Periodic Background Flush
|
||||||
|
Even if no major events happen:
|
||||||
|
- Flush every 10 seconds during active use
|
||||||
|
- Flush every 30 seconds when idle (no user input for >1 minute)
|
||||||
|
|
||||||
|
## Battery-Adaptive Flushing
|
||||||
|
|
||||||
|
Different flush strategies based on battery level:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn get_flush_interval(battery_level: f32, is_charging: bool) -> Duration {
|
||||||
|
if is_charging {
|
||||||
|
Duration::from_secs(5) // Aggressive - power available
|
||||||
|
} else if battery_level > 0.5 {
|
||||||
|
Duration::from_secs(10) // Normal
|
||||||
|
} else if battery_level > 0.2 {
|
||||||
|
Duration::from_secs(30) // Conservative
|
||||||
|
} else {
|
||||||
|
Duration::from_secs(60) // Very conservative - low battery
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**On iOS**: Use `UIDevice.current.batteryLevel` and `UIDevice.current.batteryState`
|
||||||
|
|
||||||
|
## SQLite Optimizations for Mobile
|
||||||
|
|
||||||
|
### Transaction Batching
|
||||||
|
|
||||||
|
Group multiple writes into a single transaction:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
async fn flush_write_buffer(buffer: &WriteBuffer, db: &Connection) -> Result<()> {
|
||||||
|
let tx = db.transaction()?;
|
||||||
|
|
||||||
|
// All writes in one transaction
|
||||||
|
for op in &buffer.pending_operations {
|
||||||
|
match op {
|
||||||
|
PersistenceOp::UpsertEntity { id, data } => {
|
||||||
|
tx.execute("INSERT OR REPLACE INTO entities (...) VALUES (...)", ...)?;
|
||||||
|
}
|
||||||
|
PersistenceOp::UpsertComponent { entity_id, component_type, data } => {
|
||||||
|
tx.execute("INSERT OR REPLACE INTO components (...) VALUES (...)", ...)?;
|
||||||
|
}
|
||||||
|
// ...
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit()?; // Single fsync for entire batch
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Impact**: 100 individual writes = 100 fsyncs. 1 transaction with 100 writes = 1 fsync.
|
||||||
|
|
||||||
|
### WAL Mode Checkpoint Control
|
||||||
|
|
||||||
|
```rust
|
||||||
|
async fn checkpoint_wal(db: &Connection) -> Result<()> {
|
||||||
|
// Manually checkpoint WAL to database file
|
||||||
|
db.execute("PRAGMA wal_checkpoint(PASSIVE)", [])?;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**PASSIVE checkpoint**: Doesn't block readers, syncs when possible
|
||||||
|
**When to checkpoint**: Every 10 seconds, or when WAL exceeds 1MB
|
||||||
|
|
||||||
|
### Index Strategy
|
||||||
|
|
||||||
|
Be selective about indexes - they increase write cost:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Only index what we actually query frequently
|
||||||
|
CREATE INDEX idx_components_entity ON components(entity_id);
|
||||||
|
CREATE INDEX idx_oplog_node_seq ON operation_log(node_id, sequence_number);
|
||||||
|
|
||||||
|
-- DON'T index everything just because we can
|
||||||
|
-- Every index = extra writes on every INSERT/UPDATE
|
||||||
|
```
|
||||||
|
|
||||||
|
### Page Size Optimization
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Larger page size = fewer I/O operations for sequential writes
|
||||||
|
-- Default is 4KB, but 8KB or 16KB can be better for mobile
|
||||||
|
PRAGMA page_size = 8192;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Caveat**: Must be set before database is created (or VACUUM to rebuild)
|
||||||
|
|
||||||
|
## Recovery Strategy
|
||||||
|
|
||||||
|
What happens if app crashes before flush?
|
||||||
|
|
||||||
|
### What We Lose
|
||||||
|
|
||||||
|
**Worst case**: Up to 10 seconds of component updates (positions, transforms)
|
||||||
|
|
||||||
|
**What we DON'T lose**:
|
||||||
|
- Entity existence (flushed within 1 second of creation)
|
||||||
|
- Operation log entries (flushed with vector clock updates)
|
||||||
|
- Any data from before the last checkpoint
|
||||||
|
|
||||||
|
### Recovery on Startup
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TB
|
||||||
|
A[App Starts] --> B[Open SQLite]
|
||||||
|
B --> C{Check WAL file}
|
||||||
|
C -->|WAL exists| D[Recover from WAL]
|
||||||
|
C -->|No WAL| E[Load from main DB]
|
||||||
|
D --> F[Load entities from DB]
|
||||||
|
E --> F
|
||||||
|
F --> G[Load operation log]
|
||||||
|
G --> H[Rebuild vector clock]
|
||||||
|
H --> I[Connect to gossip]
|
||||||
|
I --> J[Request sync from peers]
|
||||||
|
J --> K[Fill any gaps via anti-entropy]
|
||||||
|
K --> L[Fully recovered]
|
||||||
|
```
|
||||||
|
|
||||||
|
**Key insight**: Even if we lose local state, gossip sync repairs it. Peers send us missing operations.
|
||||||
|
|
||||||
|
### Crash Detection
|
||||||
|
|
||||||
|
On startup, detect if previous session crashed:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE session_state (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
-- On startup, check if previous session closed cleanly
|
||||||
|
SELECT value FROM session_state WHERE key = 'clean_shutdown';
|
||||||
|
|
||||||
|
-- If not found or 'false', we crashed
|
||||||
|
-- Trigger recovery procedures
|
||||||
|
```
|
||||||
|
|
||||||
|
## Platform-Specific Concerns
|
||||||
|
|
||||||
|
### iOS / iPadOS
|
||||||
|
|
||||||
|
**Background app suspension**: iOS aggressively suspends apps. We have ~5 seconds when moving to background:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// When app moves to background:
|
||||||
|
fn handle_background_event() {
|
||||||
|
// Force immediate flush
|
||||||
|
flush_write_buffer().await?;
|
||||||
|
checkpoint_wal().await?;
|
||||||
|
|
||||||
|
// Mark clean shutdown
|
||||||
|
db.execute("INSERT OR REPLACE INTO session_state VALUES ('clean_shutdown', 'true')", [])?;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Low Power Mode**: Detect and reduce flush frequency:
|
||||||
|
```swift
|
||||||
|
// iOS-specific detection
|
||||||
|
if ProcessInfo.processInfo.isLowPowerModeEnabled {
|
||||||
|
set_flush_interval(Duration::from_secs(60));
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Desktop (macOS/Linux/Windows)
|
||||||
|
|
||||||
|
More relaxed constraints:
|
||||||
|
- Battery life less critical on plugged-in desktops
|
||||||
|
- Can use more aggressive flush intervals (every 5 seconds)
|
||||||
|
- Larger WAL sizes acceptable (up to 10MB before checkpoint)
|
||||||
|
|
||||||
|
## Monitoring & Metrics
|
||||||
|
|
||||||
|
Track these metrics to tune persistence:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
struct PersistenceMetrics {
|
||||||
|
// Write volume
|
||||||
|
total_writes: u64,
|
||||||
|
bytes_written: u64,
|
||||||
|
|
||||||
|
// Timing
|
||||||
|
flush_count: u64,
|
||||||
|
avg_flush_duration: Duration,
|
||||||
|
checkpoint_count: u64,
|
||||||
|
avg_checkpoint_duration: Duration,
|
||||||
|
|
||||||
|
// WAL health
|
||||||
|
wal_size_bytes: u64,
|
||||||
|
max_wal_size_bytes: u64,
|
||||||
|
|
||||||
|
// Recovery
|
||||||
|
crash_recovery_count: u64,
|
||||||
|
clean_shutdown_count: u64,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Alerts**:
|
||||||
|
- Flush duration >50ms (disk might be slow or overloaded)
|
||||||
|
- WAL size >5MB (checkpoint more frequently)
|
||||||
|
- Crash recovery rate >10% (need more aggressive flushing)
|
||||||
|
|
||||||
|
## Write Coalescing: Deduplication
|
||||||
|
|
||||||
|
When the same entity is modified multiple times before flush, we only keep the latest:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn add_to_write_buffer(op: PersistenceOp, buffer: &mut WriteBuffer) {
|
||||||
|
match op {
|
||||||
|
PersistenceOp::UpsertComponent { entity_id, component_type, data } => {
|
||||||
|
// Remove any existing pending write for this entity+component
|
||||||
|
buffer.pending_operations.retain(|existing_op| {
|
||||||
|
!matches!(existing_op,
|
||||||
|
PersistenceOp::UpsertComponent {
|
||||||
|
entity_id: e_id,
|
||||||
|
component_type: c_type,
|
||||||
|
..
|
||||||
|
} if e_id == &entity_id && c_type == &component_type
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
// Add the new one (latest state)
|
||||||
|
buffer.pending_operations.push(op);
|
||||||
|
}
|
||||||
|
// ...
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Impact**: User drags object for 5 seconds @ 60fps = 300 transform updates → coalesced to 1 write
|
||||||
|
|
||||||
|
## Persistence vs Sync: Division of Responsibility
|
||||||
|
|
||||||
|
Important distinction:
|
||||||
|
|
||||||
|
**Persistence layer** (this RFC):
|
||||||
|
- Writes to local SQLite
|
||||||
|
- Optimized for durability and battery life
|
||||||
|
- Only cares about local state survival
|
||||||
|
|
||||||
|
**Sync layer** (RFC 0001):
|
||||||
|
- Broadcasts operations via gossip
|
||||||
|
- Maintains operation log for anti-entropy
|
||||||
|
- Ensures eventual consistency across peers
|
||||||
|
|
||||||
|
**Key insight**: These operate independently. An operation can be:
|
||||||
|
1. Logged to operation log (for sync) - happens immediately
|
||||||
|
2. Applied to ECS (for rendering) - happens immediately
|
||||||
|
3. Persisted to SQLite (for durability) - happens on flush schedule
|
||||||
|
|
||||||
|
If local state is lost due to delayed flush, sync layer repairs it from peers.
|
||||||
|
|
||||||
|
## Configuration Schema
|
||||||
|
|
||||||
|
Expose configuration for tuning:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[persistence]
|
||||||
|
# Base flush interval (may be adjusted by battery level)
|
||||||
|
flush_interval_secs = 10
|
||||||
|
|
||||||
|
# Max time to defer critical writes (entity creation, etc.)
|
||||||
|
critical_flush_delay_ms = 1000
|
||||||
|
|
||||||
|
# WAL checkpoint interval
|
||||||
|
checkpoint_interval_secs = 30
|
||||||
|
|
||||||
|
# Max WAL size before forced checkpoint
|
||||||
|
max_wal_size_mb = 5
|
||||||
|
|
||||||
|
# Adaptive flushing based on battery
|
||||||
|
battery_adaptive = true
|
||||||
|
|
||||||
|
# Flush intervals per battery tier
|
||||||
|
[persistence.battery_tiers]
|
||||||
|
charging = 5
|
||||||
|
high = 10 # >50%
|
||||||
|
medium = 30 # 20-50%
|
||||||
|
low = 60 # <20%
|
||||||
|
|
||||||
|
# Platform overrides
|
||||||
|
[persistence.ios]
|
||||||
|
background_flush_timeout_secs = 5
|
||||||
|
low_power_mode_interval_secs = 60
|
||||||
|
```
|
||||||
|
|
||||||
|
## Example System Implementation
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn persistence_system(
|
||||||
|
dirty: Res<DirtyEntities>,
|
||||||
|
mut write_buffer: ResMut<WriteBuffer>,
|
||||||
|
db: Res<DatabaseConnection>,
|
||||||
|
time: Res<Time>,
|
||||||
|
battery: Res<BatteryStatus>,
|
||||||
|
query: Query<(Entity, &NetworkedEntity, &Transform, &/* other components */)>,
|
||||||
|
) {
|
||||||
|
// Step 1: Check if it's time to collect dirty entities
|
||||||
|
let flush_interval = get_flush_interval(battery.level, battery.is_charging);
|
||||||
|
|
||||||
|
if time.elapsed() - write_buffer.last_flush < flush_interval {
|
||||||
|
return; // Not time yet
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: Collect dirty entities into write buffer
|
||||||
|
for entity_uuid in &dirty.entities {
|
||||||
|
if let Some((entity, net_entity, transform, /* ... */)) =
|
||||||
|
query.iter().find(|(_, ne, ..)| ne.network_id == *entity_uuid)
|
||||||
|
{
|
||||||
|
// Serialize component
|
||||||
|
let transform_data = bincode::serialize(transform)?;
|
||||||
|
|
||||||
|
// Add to write buffer (coalescing happens here)
|
||||||
|
write_buffer.add(PersistenceOp::UpsertComponent {
|
||||||
|
entity_id: *entity_uuid,
|
||||||
|
component_type: "Transform".to_string(),
|
||||||
|
data: transform_data,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: Flush write buffer to SQLite (async, non-blocking)
|
||||||
|
if write_buffer.pending_operations.len() > 0 {
|
||||||
|
let ops = std::mem::take(&mut write_buffer.pending_operations);
|
||||||
|
|
||||||
|
// Spawn async task to write to SQLite
|
||||||
|
spawn_blocking(move || {
|
||||||
|
flush_to_sqlite(&ops, &db)
|
||||||
|
});
|
||||||
|
|
||||||
|
write_buffer.last_flush = time.elapsed();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: Clear dirty tracking (they're now in write buffer/SQLite)
|
||||||
|
dirty.entities.clear();
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Trade-offs and Decisions
|
||||||
|
|
||||||
|
### Why WAL Mode?
|
||||||
|
|
||||||
|
**Alternatives**:
|
||||||
|
- DELETE mode (traditional journaling)
|
||||||
|
- MEMORY mode (no durability)
|
||||||
|
|
||||||
|
**Decision**: WAL mode because:
|
||||||
|
- Better write concurrency (readers don't block writers)
|
||||||
|
- Fewer `fsync()` calls (only on checkpoint)
|
||||||
|
- Better crash recovery (WAL can be replayed)
|
||||||
|
|
||||||
|
### Why Not Use a Dirty Flag on Components?
|
||||||
|
|
||||||
|
We could mark components with a `#[derive(Dirty)]` flag, but:
|
||||||
|
- Bevy's `Changed<T>` already gives us change detection for free
|
||||||
|
- A separate dirty flag adds memory overhead
|
||||||
|
- We'd need to manually clear flags after persistence
|
||||||
|
|
||||||
|
**Decision**: Use Bevy's change detection + our own dirty tracking resource
|
||||||
|
|
||||||
|
### Why Not Use a Separate Persistence Thread?
|
||||||
|
|
||||||
|
We could run SQLite writes on a dedicated thread:
|
||||||
|
|
||||||
|
**Pros**: Never blocks main thread
|
||||||
|
**Cons**: More complex synchronization, harder to guarantee flush order
|
||||||
|
|
||||||
|
**Decision**: Use `spawn_blocking` from async runtime (Tokio). Simpler, good enough.
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. **Write ordering**: Do we need to guarantee operation log entries are persisted before entity state? Or can they be out of order?
|
||||||
|
2. **Compression**: Should we compress component data before writing to SQLite? Trade-off: CPU vs I/O
|
||||||
|
3. **Memory limits**: On iPad with 2GB RAM, how large can the write buffer grow before we force a flush?
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
We'll know this is working when:
|
||||||
|
- [ ] App can run for 30 minutes with <5% battery drain attributed to persistence
|
||||||
|
- [ ] Crash recovery loses <10 seconds of work
|
||||||
|
- [ ] No perceptible frame drops during flush operations
|
||||||
|
- [ ] SQLite file size grows linearly with user data, not explosively
|
||||||
|
- [ ] WAL checkpoints complete in <100ms
|
||||||
|
|
||||||
|
## Implementation Phases
|
||||||
|
|
||||||
|
1. **Phase 1**: Basic in-memory dirty tracking + batched writes
|
||||||
|
2. **Phase 2**: WAL mode + manual checkpoint control
|
||||||
|
3. **Phase 3**: Battery-adaptive flushing
|
||||||
|
4. **Phase 4**: iOS background handling
|
||||||
|
5. **Phase 5**: Monitoring and tuning based on metrics
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- [SQLite WAL Mode](https://www.sqlite.org/wal.html)
|
||||||
|
- [iOS Background Execution](https://developer.apple.com/documentation/uikit/app_and_environment/scenes/preparing_your_ui_to_run_in_the_background)
|
||||||
|
- [Bevy Change Detection](https://docs.rs/bevy/latest/bevy/ecs/change_detection/)
|
||||||
39
docs/rfcs/README.md
Normal file
39
docs/rfcs/README.md
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
# RFCs
|
||||||
|
|
||||||
|
Request for Comments (RFCs) for major design decisions in the Lonni project.
|
||||||
|
|
||||||
|
## Active RFCs
|
||||||
|
|
||||||
|
- [RFC 0001: CRDT Synchronization Protocol over iroh-gossip](./0001-crdt-gossip-sync.md) - Draft
|
||||||
|
|
||||||
|
## RFC Process
|
||||||
|
|
||||||
|
1. **Draft**: Initial proposal, open for discussion
|
||||||
|
2. **Review**: Team reviews and provides feedback
|
||||||
|
3. **Accepted**: Approved for implementation
|
||||||
|
4. **Implemented**: Design has been built
|
||||||
|
5. **Superseded**: Replaced by a newer RFC
|
||||||
|
|
||||||
|
RFCs are living documents - they can be updated as we learn during implementation.
|
||||||
|
|
||||||
|
## When to Write an RFC
|
||||||
|
|
||||||
|
Write an RFC when:
|
||||||
|
- Making architectural decisions that affect multiple parts of the system
|
||||||
|
- Choosing between significantly different approaches
|
||||||
|
- Introducing new protocols or APIs
|
||||||
|
- Making breaking changes
|
||||||
|
|
||||||
|
Don't write an RFC for:
|
||||||
|
- Small bug fixes
|
||||||
|
- Minor refactors
|
||||||
|
- Isolated feature additions
|
||||||
|
- Experimental prototypes
|
||||||
|
|
||||||
|
## RFC Format
|
||||||
|
|
||||||
|
- **Narrative first**: Tell the story of why and how
|
||||||
|
- **Explain trade-offs**: What alternatives were considered?
|
||||||
|
- **API examples**: Show how it would be used (not full implementations)
|
||||||
|
- **Open questions**: What's still unclear?
|
||||||
|
- **Success criteria**: How do we know it works?
|
||||||
640
index.html
Normal file
640
index.html
Normal file
@@ -0,0 +1,640 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>go_emotions Gradient Space - OKLab Edition</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
padding: 20px;
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
|
||||||
|
background: #1a1a1a;
|
||||||
|
color: #fff;
|
||||||
|
height: 100vh;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.container {
|
||||||
|
max-width: 1600px;
|
||||||
|
margin: 0 auto;
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr 350px;
|
||||||
|
gap: 20px;
|
||||||
|
height: calc(100vh - 40px);
|
||||||
|
}
|
||||||
|
.main-area {
|
||||||
|
min-width: 0;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
.controls {
|
||||||
|
background: #2a2a2a;
|
||||||
|
padding: 20px;
|
||||||
|
border-radius: 8px;
|
||||||
|
max-height: calc(100vh - 40px);
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
h1 {
|
||||||
|
margin-bottom: 10px;
|
||||||
|
font-size: 24px;
|
||||||
|
}
|
||||||
|
.subtitle {
|
||||||
|
margin-bottom: 20px;
|
||||||
|
color: #aaa;
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
canvas {
|
||||||
|
display: block;
|
||||||
|
margin: 20px auto;
|
||||||
|
border: 1px solid #444;
|
||||||
|
cursor: crosshair;
|
||||||
|
touch-action: none;
|
||||||
|
background: #000;
|
||||||
|
}
|
||||||
|
canvas.dragging {
|
||||||
|
cursor: move !important;
|
||||||
|
}
|
||||||
|
canvas.hovering {
|
||||||
|
cursor: grab;
|
||||||
|
}
|
||||||
|
.info {
|
||||||
|
margin-top: 20px;
|
||||||
|
padding: 15px;
|
||||||
|
background: #2a2a2a;
|
||||||
|
border-radius: 8px;
|
||||||
|
font-family: monospace;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
.weights {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
||||||
|
gap: 5px;
|
||||||
|
margin-top: 10px;
|
||||||
|
}
|
||||||
|
.weight-item {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
}
|
||||||
|
.weight-bar {
|
||||||
|
height: 4px;
|
||||||
|
background: #555;
|
||||||
|
margin-top: 2px;
|
||||||
|
}
|
||||||
|
.weight-fill {
|
||||||
|
height: 100%;
|
||||||
|
background: #4FC3F7;
|
||||||
|
}
|
||||||
|
.emotion-control {
|
||||||
|
margin-bottom: 15px;
|
||||||
|
padding: 10px;
|
||||||
|
background: #1a1a1a;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
.emotion-control label {
|
||||||
|
display: block;
|
||||||
|
font-size: 12px;
|
||||||
|
margin-bottom: 5px;
|
||||||
|
text-transform: capitalize;
|
||||||
|
}
|
||||||
|
.emotion-control input[type="color"] {
|
||||||
|
width: 100%;
|
||||||
|
height: 30px;
|
||||||
|
border: none;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.export-btn {
|
||||||
|
width: 100%;
|
||||||
|
padding: 12px;
|
||||||
|
background: #4FC3F7;
|
||||||
|
color: #000;
|
||||||
|
border: none;
|
||||||
|
border-radius: 4px;
|
||||||
|
font-weight: bold;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 14px;
|
||||||
|
margin-bottom: 20px;
|
||||||
|
}
|
||||||
|
.export-btn:hover {
|
||||||
|
background: #6FD3FF;
|
||||||
|
}
|
||||||
|
.controls h2 {
|
||||||
|
font-size: 16px;
|
||||||
|
margin-bottom: 15px;
|
||||||
|
}
|
||||||
|
.hint {
|
||||||
|
font-size: 11px;
|
||||||
|
color: #888;
|
||||||
|
margin-top: 5px;
|
||||||
|
}
|
||||||
|
.loading-spinner {
|
||||||
|
position: fixed;
|
||||||
|
top: 50%;
|
||||||
|
left: 50%;
|
||||||
|
transform: translate(-50%, -50%);
|
||||||
|
z-index: 1000;
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
.loading-spinner.active {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
.spinner-circle {
|
||||||
|
width: 50px;
|
||||||
|
height: 50px;
|
||||||
|
border: 4px solid rgba(79, 195, 247, 0.2);
|
||||||
|
border-top-color: #4FC3F7;
|
||||||
|
border-radius: 50%;
|
||||||
|
animation: spin 0.8s linear infinite;
|
||||||
|
}
|
||||||
|
.spinner-text {
|
||||||
|
color: #4FC3F7;
|
||||||
|
font-size: 14px;
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
@keyframes spin {
|
||||||
|
to { transform: rotate(360deg); }
|
||||||
|
}
|
||||||
|
.loading-overlay {
|
||||||
|
position: fixed;
|
||||||
|
top: 0;
|
||||||
|
left: 0;
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
background: rgba(0, 0, 0, 0.3);
|
||||||
|
z-index: 999;
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
.loading-overlay.active {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="loading-overlay" id="loadingOverlay"></div>
|
||||||
|
<div class="loading-spinner" id="loadingSpinner">
|
||||||
|
<div class="spinner-circle"></div>
|
||||||
|
<div class="spinner-text">Calculating gradient...</div>
|
||||||
|
</div>
|
||||||
|
<div class="container">
|
||||||
|
<div class="main-area">
|
||||||
|
<h1>go_emotions Gradient Space - OKLab Edition</h1>
|
||||||
|
<div class="subtitle">Drag centroids to reposition emotions. Colors blend in perceptually uniform OKLab space.</div>
|
||||||
|
<canvas id="gradientCanvas" width="800" height="800"></canvas>
|
||||||
|
<div class="info">
|
||||||
|
<div>Hover to see emotion weights | Click and drag centroids to move</div>
|
||||||
|
<div id="coordinates" style="margin-top: 5px;">Position: (-, -)</div>
|
||||||
|
<div class="weights" id="weights"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="controls">
|
||||||
|
<button class="export-btn" onclick="exportConfiguration()">Export Configuration</button>
|
||||||
|
<h2>Emotion Colors</h2>
|
||||||
|
<div class="hint">Click to edit colors for each emotion</div>
|
||||||
|
<div id="colorControls"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
// OKLab color space conversion functions
|
||||||
|
// sRGB to Linear RGB
|
||||||
|
function srgbToLinear(c) {
|
||||||
|
const abs = Math.abs(c);
|
||||||
|
if (abs <= 0.04045) {
|
||||||
|
return c / 12.92;
|
||||||
|
}
|
||||||
|
return Math.sign(c) * Math.pow((abs + 0.055) / 1.055, 2.4);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Linear RGB to sRGB
|
||||||
|
function linearToSrgb(c) {
|
||||||
|
const abs = Math.abs(c);
|
||||||
|
if (abs <= 0.0031308) {
|
||||||
|
return c * 12.92;
|
||||||
|
}
|
||||||
|
return Math.sign(c) * (1.055 * Math.pow(abs, 1 / 2.4) - 0.055);
|
||||||
|
}
|
||||||
|
|
||||||
|
// RGB (0-255) to OKLab
|
||||||
|
function rgbToOklab(r, g, b) {
|
||||||
|
// Normalize to 0-1
|
||||||
|
r = r / 255;
|
||||||
|
g = g / 255;
|
||||||
|
b = b / 255;
|
||||||
|
|
||||||
|
// Convert to linear RGB
|
||||||
|
r = srgbToLinear(r);
|
||||||
|
g = srgbToLinear(g);
|
||||||
|
b = srgbToLinear(b);
|
||||||
|
|
||||||
|
// Linear RGB to LMS
|
||||||
|
const l = 0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b;
|
||||||
|
const m = 0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b;
|
||||||
|
const s = 0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b;
|
||||||
|
|
||||||
|
// LMS to OKLab
|
||||||
|
const l_ = Math.cbrt(l);
|
||||||
|
const m_ = Math.cbrt(m);
|
||||||
|
const s_ = Math.cbrt(s);
|
||||||
|
|
||||||
|
return {
|
||||||
|
L: 0.2104542553 * l_ + 0.7936177850 * m_ - 0.0040720468 * s_,
|
||||||
|
a: 1.9779984951 * l_ - 2.4285922050 * m_ + 0.4505937099 * s_,
|
||||||
|
b: 0.0259040371 * l_ + 0.7827717662 * m_ - 0.8086757660 * s_
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// OKLab to RGB (0-255)
|
||||||
|
function oklabToRgb(L, a, b) {
|
||||||
|
// OKLab to LMS
|
||||||
|
const l_ = L + 0.3963377774 * a + 0.2158037573 * b;
|
||||||
|
const m_ = L - 0.1055613458 * a - 0.0638541728 * b;
|
||||||
|
const s_ = L - 0.0894841775 * a - 1.2914855480 * b;
|
||||||
|
|
||||||
|
const l = l_ * l_ * l_;
|
||||||
|
const m = m_ * m_ * m_;
|
||||||
|
const s = s_ * s_ * s_;
|
||||||
|
|
||||||
|
// LMS to linear RGB
|
||||||
|
let r = +4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s;
|
||||||
|
let g = -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s;
|
||||||
|
let b_ = -0.0041960863 * l - 0.7034186147 * m + 1.7076147010 * s;
|
||||||
|
|
||||||
|
// Linear RGB to sRGB
|
||||||
|
r = linearToSrgb(r);
|
||||||
|
g = linearToSrgb(g);
|
||||||
|
b_ = linearToSrgb(b_);
|
||||||
|
|
||||||
|
// Clamp and convert to 0-255
|
||||||
|
r = Math.max(0, Math.min(1, r)) * 255;
|
||||||
|
g = Math.max(0, Math.min(1, g)) * 255;
|
||||||
|
b_ = Math.max(0, Math.min(1, b_)) * 255;
|
||||||
|
|
||||||
|
return [r, g, b_];
|
||||||
|
}
|
||||||
|
|
||||||
|
const emotions = [
|
||||||
|
{ name: 'admiration', color: [255, 107, 107] },
|
||||||
|
{ name: 'amusement', color: [255, 217, 61] },
|
||||||
|
{ name: 'anger', color: [211, 47, 47] },
|
||||||
|
{ name: 'annoyance', color: [245, 124, 0] },
|
||||||
|
{ name: 'approval', color: [102, 187, 106] },
|
||||||
|
{ name: 'caring', color: [255, 182, 193] },
|
||||||
|
{ name: 'confusion', color: [156, 39, 176] },
|
||||||
|
{ name: 'curiosity', color: [79, 195, 247] },
|
||||||
|
{ name: 'desire', color: [233, 30, 99] },
|
||||||
|
{ name: 'disappointment', color: [109, 76, 65] },
|
||||||
|
{ name: 'disapproval', color: [139, 69, 19] },
|
||||||
|
{ name: 'disgust', color: [85, 139, 47] },
|
||||||
|
{ name: 'embarrassment', color: [255, 152, 0] },
|
||||||
|
{ name: 'excitement', color: [255, 241, 118] },
|
||||||
|
{ name: 'fear', color: [66, 66, 66] },
|
||||||
|
{ name: 'gratitude', color: [255, 224, 130] },
|
||||||
|
{ name: 'grief', color: [55, 71, 79] },
|
||||||
|
{ name: 'joy', color: [255, 235, 59] },
|
||||||
|
{ name: 'love', color: [255, 64, 129] },
|
||||||
|
{ name: 'nervousness', color: [126, 87, 194] },
|
||||||
|
{ name: 'optimism', color: [129, 199, 132] },
|
||||||
|
{ name: 'pride', color: [255, 213, 79] },
|
||||||
|
{ name: 'realization', color: [77, 208, 225] },
|
||||||
|
{ name: 'relief', color: [174, 213, 129] },
|
||||||
|
{ name: 'remorse', color: [186, 104, 200] },
|
||||||
|
{ name: 'sadness', color: [92, 107, 192] },
|
||||||
|
{ name: 'surprise', color: [255, 111, 0] },
|
||||||
|
{ name: 'neutral', color: [144, 164, 174] }
|
||||||
|
];
|
||||||
|
|
||||||
|
const canvas = document.getElementById('gradientCanvas');
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
const width = canvas.width;
|
||||||
|
const height = canvas.height;
|
||||||
|
const centerX = width / 2;
|
||||||
|
const centerY = height / 2;
|
||||||
|
const radius = Math.min(width, height) * 0.4;
|
||||||
|
|
||||||
|
// Clear canvas to black initially
|
||||||
|
ctx.fillStyle = '#000000';
|
||||||
|
ctx.fillRect(0, 0, width, height);
|
||||||
|
|
||||||
|
// Position emotions in a circle
|
||||||
|
emotions.forEach((emotion, i) => {
|
||||||
|
const angle = (i / emotions.length) * Math.PI * 2;
|
||||||
|
emotion.x = centerX + Math.cos(angle) * radius;
|
||||||
|
emotion.y = centerY + Math.sin(angle) * radius;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Dragging state
|
||||||
|
let draggedEmotion = null;
|
||||||
|
let isDragging = false;
|
||||||
|
let gradientImageData = null;
|
||||||
|
let animationFrameId = null;
|
||||||
|
let pendingUpdate = false;
|
||||||
|
|
||||||
|
// Initialize color controls
|
||||||
|
function initColorControls() {
|
||||||
|
const controlsDiv = document.getElementById('colorControls');
|
||||||
|
emotions.forEach((emotion, idx) => {
|
||||||
|
const div = document.createElement('div');
|
||||||
|
div.className = 'emotion-control';
|
||||||
|
|
||||||
|
const label = document.createElement('label');
|
||||||
|
label.textContent = emotion.name;
|
||||||
|
|
||||||
|
const input = document.createElement('input');
|
||||||
|
input.type = 'color';
|
||||||
|
input.id = `color-${idx}`;
|
||||||
|
const hexColor = `#${emotion.color.map(c => Math.round(c).toString(16).padStart(2, '0')).join('')}`;
|
||||||
|
input.value = hexColor;
|
||||||
|
|
||||||
|
const updateColor = (e) => {
|
||||||
|
const hex = e.target.value;
|
||||||
|
const r = parseInt(hex.substring(1, 3), 16);
|
||||||
|
const g = parseInt(hex.substring(3, 5), 16);
|
||||||
|
const b = parseInt(hex.substring(5, 7), 16);
|
||||||
|
emotions[idx].color = [r, g, b];
|
||||||
|
redrawGradient();
|
||||||
|
};
|
||||||
|
|
||||||
|
input.addEventListener('input', updateColor);
|
||||||
|
input.addEventListener('change', updateColor);
|
||||||
|
|
||||||
|
div.appendChild(label);
|
||||||
|
div.appendChild(input);
|
||||||
|
controlsDiv.appendChild(div);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loading indicator helpers
|
||||||
|
function showLoading() {
|
||||||
|
document.getElementById('loadingOverlay').classList.add('active');
|
||||||
|
document.getElementById('loadingSpinner').classList.add('active');
|
||||||
|
}
|
||||||
|
|
||||||
|
function hideLoading() {
|
||||||
|
document.getElementById('loadingOverlay').classList.remove('active');
|
||||||
|
document.getElementById('loadingSpinner').classList.remove('active');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate and cache the gradient
|
||||||
|
function calculateGradient() {
|
||||||
|
const imageData = ctx.createImageData(width, height);
|
||||||
|
const data = imageData.data;
|
||||||
|
|
||||||
|
for (let y = 0; y < height; y++) {
|
||||||
|
for (let x = 0; x < width; x++) {
|
||||||
|
const idx = (y * width + x) * 4;
|
||||||
|
|
||||||
|
// Calculate weights using inverse distance
|
||||||
|
let totalWeight = 0;
|
||||||
|
const weights = [];
|
||||||
|
|
||||||
|
emotions.forEach(emotion => {
|
||||||
|
const dx = x - emotion.x;
|
||||||
|
const dy = y - emotion.y;
|
||||||
|
const dist = Math.sqrt(dx * dx + dy * dy);
|
||||||
|
const weight = 1 / (Math.pow(dist, 2.5) + 1);
|
||||||
|
weights.push(weight);
|
||||||
|
totalWeight += weight;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Normalize weights and blend colors in OKLab space
|
||||||
|
let L = 0, a = 0, b = 0;
|
||||||
|
weights.forEach((weight, i) => {
|
||||||
|
const normalizedWeight = weight / totalWeight;
|
||||||
|
const lab = rgbToOklab(...emotions[i].color);
|
||||||
|
L += lab.L * normalizedWeight;
|
||||||
|
a += lab.a * normalizedWeight;
|
||||||
|
b += lab.b * normalizedWeight;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Convert back to RGB
|
||||||
|
const [r, g, b_] = oklabToRgb(L, a, b);
|
||||||
|
|
||||||
|
data[idx] = r;
|
||||||
|
data[idx + 1] = g;
|
||||||
|
data[idx + 2] = b_;
|
||||||
|
data[idx + 3] = 255;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
gradientImageData = imageData;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Redraw the entire gradient
|
||||||
|
function redrawGradient() {
|
||||||
|
showLoading();
|
||||||
|
// Use setTimeout to allow the loading spinner to render before blocking
|
||||||
|
setTimeout(() => {
|
||||||
|
calculateGradient();
|
||||||
|
renderCanvas();
|
||||||
|
hideLoading();
|
||||||
|
}, 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render the canvas (gradient + points)
|
||||||
|
function renderCanvas() {
|
||||||
|
ctx.putImageData(gradientImageData, 0, 0);
|
||||||
|
drawEmotionPoints();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Schedule a render using requestAnimationFrame
|
||||||
|
function scheduleRender() {
|
||||||
|
if (!pendingUpdate) {
|
||||||
|
pendingUpdate = true;
|
||||||
|
requestAnimationFrame(() => {
|
||||||
|
renderCanvas();
|
||||||
|
pendingUpdate = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Draw emotion labels and centroids
|
||||||
|
function drawEmotionPoints() {
|
||||||
|
ctx.font = '12px monospace';
|
||||||
|
ctx.textAlign = 'center';
|
||||||
|
ctx.textBaseline = 'middle';
|
||||||
|
|
||||||
|
emotions.forEach((emotion, i) => {
|
||||||
|
// Draw a larger circle at each emotion point for better dragging
|
||||||
|
ctx.fillStyle = `rgb(${emotion.color[0]}, ${emotion.color[1]}, ${emotion.color[2]})`;
|
||||||
|
ctx.strokeStyle = '#fff';
|
||||||
|
ctx.lineWidth = 2;
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.arc(emotion.x, emotion.y, 8, 0, Math.PI * 2);
|
||||||
|
ctx.fill();
|
||||||
|
ctx.stroke();
|
||||||
|
|
||||||
|
// Draw label with background
|
||||||
|
const dx = emotion.x - centerX;
|
||||||
|
const dy = emotion.y - centerY;
|
||||||
|
const angle = Math.atan2(dy, dx);
|
||||||
|
const labelRadius = Math.sqrt(dx * dx + dy * dy) + 30;
|
||||||
|
const labelX = centerX + Math.cos(angle) * labelRadius;
|
||||||
|
const labelY = centerY + Math.sin(angle) * labelRadius;
|
||||||
|
|
||||||
|
ctx.fillStyle = 'rgba(0, 0, 0, 0.7)';
|
||||||
|
const textWidth = ctx.measureText(emotion.name).width;
|
||||||
|
ctx.fillRect(labelX - textWidth/2 - 3, labelY - 8, textWidth + 6, 16);
|
||||||
|
|
||||||
|
ctx.fillStyle = '#fff';
|
||||||
|
ctx.fillText(emotion.name, labelX, labelY);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mouse event handlers
|
||||||
|
canvas.addEventListener('mousedown', (e) => {
|
||||||
|
const rect = canvas.getBoundingClientRect();
|
||||||
|
const x = (e.clientX - rect.left) * (canvas.width / rect.width);
|
||||||
|
const y = (e.clientY - rect.top) * (canvas.height / rect.height);
|
||||||
|
|
||||||
|
// Check if clicking on any emotion centroid (larger hit area for easier clicking)
|
||||||
|
for (const emotion of emotions) {
|
||||||
|
const dx = x - emotion.x;
|
||||||
|
const dy = y - emotion.y;
|
||||||
|
const dist = Math.sqrt(dx * dx + dy * dy);
|
||||||
|
if (dist < 25) { // Increased from 15 to 25 for easier clicking
|
||||||
|
draggedEmotion = emotion;
|
||||||
|
isDragging = true;
|
||||||
|
canvas.classList.add('dragging');
|
||||||
|
e.preventDefault();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
canvas.addEventListener('mousemove', (e) => {
|
||||||
|
const rect = canvas.getBoundingClientRect();
|
||||||
|
const x = (e.clientX - rect.left) * (canvas.width / rect.width);
|
||||||
|
const y = (e.clientY - rect.top) * (canvas.height / rect.height);
|
||||||
|
|
||||||
|
if (isDragging && draggedEmotion) {
|
||||||
|
e.preventDefault();
|
||||||
|
draggedEmotion.x = Math.max(0, Math.min(width, x));
|
||||||
|
draggedEmotion.y = Math.max(0, Math.min(height, y));
|
||||||
|
// Use requestAnimationFrame for smooth updates
|
||||||
|
scheduleRender();
|
||||||
|
} else {
|
||||||
|
// Check if hovering over any centroid
|
||||||
|
let isHovering = false;
|
||||||
|
for (const emotion of emotions) {
|
||||||
|
const dx = x - emotion.x;
|
||||||
|
const dy = y - emotion.y;
|
||||||
|
const dist = Math.sqrt(dx * dx + dy * dy);
|
||||||
|
if (dist < 25) {
|
||||||
|
isHovering = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update cursor
|
||||||
|
if (isHovering) {
|
||||||
|
canvas.classList.add('hovering');
|
||||||
|
} else {
|
||||||
|
canvas.classList.remove('hovering');
|
||||||
|
}
|
||||||
|
|
||||||
|
showWeights(Math.floor(x), Math.floor(y));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
canvas.addEventListener('mouseup', () => {
|
||||||
|
if (isDragging) {
|
||||||
|
// Recalculate gradient when drag ends
|
||||||
|
redrawGradient();
|
||||||
|
}
|
||||||
|
isDragging = false;
|
||||||
|
draggedEmotion = null;
|
||||||
|
canvas.classList.remove('dragging');
|
||||||
|
canvas.classList.remove('hovering');
|
||||||
|
});
|
||||||
|
|
||||||
|
canvas.addEventListener('mouseleave', () => {
|
||||||
|
if (isDragging) {
|
||||||
|
// Recalculate gradient when drag ends
|
||||||
|
redrawGradient();
|
||||||
|
}
|
||||||
|
isDragging = false;
|
||||||
|
draggedEmotion = null;
|
||||||
|
canvas.classList.remove('dragging');
|
||||||
|
canvas.classList.remove('hovering');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Interactive hover/click
|
||||||
|
function showWeights(x, y) {
|
||||||
|
const coordDiv = document.getElementById('coordinates');
|
||||||
|
const weightsDiv = document.getElementById('weights');
|
||||||
|
|
||||||
|
coordDiv.textContent = `Position: (${x}, ${y})`;
|
||||||
|
|
||||||
|
// Calculate weights for this position
|
||||||
|
let totalWeight = 0;
|
||||||
|
const weights = [];
|
||||||
|
|
||||||
|
emotions.forEach(emotion => {
|
||||||
|
const dx = x - emotion.x;
|
||||||
|
const dy = y - emotion.y;
|
||||||
|
const dist = Math.sqrt(dx * dx + dy * dy);
|
||||||
|
const weight = 1 / (Math.pow(dist, 2.5) + 1);
|
||||||
|
weights.push(weight);
|
||||||
|
totalWeight += weight;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sort by weight descending
|
||||||
|
const sortedEmotions = emotions.map((e, i) => ({
|
||||||
|
name: e.name,
|
||||||
|
weight: weights[i] / totalWeight
|
||||||
|
})).sort((a, b) => b.weight - a.weight);
|
||||||
|
|
||||||
|
weightsDiv.innerHTML = sortedEmotions
|
||||||
|
.filter(e => e.weight > 0.01)
|
||||||
|
.map(e => `
|
||||||
|
<div>
|
||||||
|
<div class="weight-item">
|
||||||
|
<span>${e.name}</span>
|
||||||
|
<span>${(e.weight * 100).toFixed(1)}%</span>
|
||||||
|
</div>
|
||||||
|
<div class="weight-bar">
|
||||||
|
<div class="weight-fill" style="width: ${e.weight * 100}%"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`).join('');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export configuration
|
||||||
|
function exportConfiguration() {
|
||||||
|
const config = {
|
||||||
|
colorSpace: 'oklab',
|
||||||
|
canvasSize: { width, height },
|
||||||
|
emotions: emotions.map(e => ({
|
||||||
|
name: e.name,
|
||||||
|
position: { x: e.x, y: e.y },
|
||||||
|
color: { r: e.color[0], g: e.color[1], b: e.color[2] }
|
||||||
|
})),
|
||||||
|
metadata: {
|
||||||
|
exportDate: new Date().toISOString(),
|
||||||
|
version: '1.0'
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const dataStr = JSON.stringify(config, null, 2);
|
||||||
|
const dataBlob = new Blob([dataStr], { type: 'application/json' });
|
||||||
|
const url = URL.createObjectURL(dataBlob);
|
||||||
|
|
||||||
|
const link = document.createElement('a');
|
||||||
|
link.href = url;
|
||||||
|
link.download = `emotion-gradient-config-${Date.now()}.json`;
|
||||||
|
document.body.appendChild(link);
|
||||||
|
link.click();
|
||||||
|
document.body.removeChild(link);
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
|
||||||
|
console.log('Configuration exported:', config);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize
|
||||||
|
initColorControls();
|
||||||
|
redrawGradient();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Reference in New Issue
Block a user