initial commit

Signed-off-by: Sienna Meridian Satterwhite <sienna@r3t.io>
This commit is contained in:
2025-11-15 23:42:12 +00:00
commit 2bad250a04
47 changed files with 14645 additions and 0 deletions

4
crates/lib/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
/target
chat.db
*.db-shm
*.db-wal

20
crates/lib/Cargo.toml Normal file
View File

@@ -0,0 +1,20 @@
[package]
name = "lib"
version = "0.1.0"
edition.workspace = true
[dependencies]
rusqlite = { version = "0.37.0", features = ["bundled"] }
chrono = { version = "0.4", features = ["serde"] }
thiserror = "2.0"
serde = { version = "1.0", features = ["derive"] }
serde_json.workspace = true
crdts.workspace = true
anyhow.workspace = true
sync-macros = { path = "../sync-macros" }
[dev-dependencies]
tokio.workspace = true
iroh.workspace = true
iroh-gossip.workspace = true
futures-lite = "2.0"

139
crates/lib/src/db.rs Normal file
View File

@@ -0,0 +1,139 @@
use crate::error::Result;
use crate::models::*;
use rusqlite::{Connection, OpenFlags, Row, params};
pub struct ChatDb {
conn: Connection,
}
impl ChatDb {
/// Open a connection to the chat database in read-only mode
pub fn open(path: &str) -> Result<Self> {
let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
Ok(Self { conn })
}
/// Get messages from the conversation with +31 6 39 13 29 13
///
/// Returns messages from January 1, 2024 to present from the conversation
/// with the specified Dutch phone number.
///
/// # Arguments
///
/// * `start_date` - Start date (defaults to January 1, 2024 if None)
/// * `end_date` - End date (defaults to current time if None)
pub fn get_our_messages(
&self,
start_date: Option<chrono::DateTime<chrono::Utc>>,
end_date: Option<chrono::DateTime<chrono::Utc>>,
) -> Result<Vec<Message>> {
use chrono::{TimeZone, Utc};
// Default date range: January 1, 2024 to now
let start =
start_date.unwrap_or_else(|| Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap());
let end = end_date.unwrap_or_else(|| Utc::now());
// Convert to Apple timestamps (nanoseconds since 2001-01-01)
let start_timestamp = datetime_to_apple_timestamp(start);
let end_timestamp = datetime_to_apple_timestamp(end);
// The phone number might be stored with or without spaces
let phone_with_spaces = "+31 6 39 13 29 13";
let phone_without_spaces = "+31639132913";
// Find the chat with this phone number (try both formats)
let chat = self
.get_chat_for_phone_number(phone_with_spaces)
.or_else(|_| self.get_chat_for_phone_number(phone_without_spaces))?;
// Get messages from this chat within the date range
let mut stmt = self.conn.prepare(
"SELECT m.ROWID, m.guid, m.text, m.service, m.handle_id, m.date, m.date_read, m.date_delivered,
m.is_from_me, m.is_read, m.is_delivered, m.is_sent, m.is_emote, m.is_audio_message,
m.cache_has_attachments, m.associated_message_guid, m.associated_message_type,
m.thread_originator_guid, m.reply_to_guid, m.is_spam
FROM message m
INNER JOIN chat_message_join cmj ON m.ROWID = cmj.message_id
WHERE cmj.chat_id = ?
AND m.date >= ?
AND m.date <= ?
ORDER BY m.date ASC"
)?;
let messages = stmt
.query_map(
params![chat.rowid, start_timestamp, end_timestamp],
map_message_row,
)?
.collect::<std::result::Result<Vec<_>, _>>()?;
Ok(messages)
}
/// Helper function to find the largest chat with a specific phone number
fn get_chat_for_phone_number(&self, phone_number: &str) -> Result<Chat> {
let mut stmt = self.conn.prepare(
"SELECT c.ROWID, c.guid, c.chat_identifier, c.service_name, c.display_name,
c.group_id, c.room_name, c.is_archived, c.is_filtered,
c.last_read_message_timestamp, COUNT(cmj.message_id) as msg_count
FROM chat c
INNER JOIN chat_handle_join chj ON c.ROWID = chj.chat_id
INNER JOIN handle h ON chj.handle_id = h.ROWID
INNER JOIN chat_message_join cmj ON c.ROWID = cmj.chat_id
WHERE h.id = ?
GROUP BY c.ROWID
ORDER BY msg_count DESC
LIMIT 1"
)?;
let chat = stmt.query_row(params![phone_number], |row| {
Ok(Chat {
rowid: row.get(0)?,
guid: row.get(1)?,
chat_identifier: row.get(2)?,
service_name: row.get(3)?,
display_name: row.get(4)?,
group_id: row.get(5)?,
room_name: row.get(6)?,
is_archived: row.get::<_, i64>(7)? != 0,
is_filtered: row.get::<_, i64>(8)? != 0,
last_read_message_timestamp: row.get::<_, Option<i64>>(9)?.map(apple_timestamp_to_datetime),
})
})?;
Ok(chat)
}
}
// Helper function to map database rows to structs
fn map_message_row(row: &Row) -> rusqlite::Result<Message> {
Ok(Message {
rowid: row.get(0)?,
guid: row.get(1)?,
text: row.get(2)?,
service: row.get(3)?,
handle_id: row.get(4)?,
date: row
.get::<_, Option<i64>>(5)?
.map(apple_timestamp_to_datetime),
date_read: row
.get::<_, Option<i64>>(6)?
.map(apple_timestamp_to_datetime),
date_delivered: row
.get::<_, Option<i64>>(7)?
.map(apple_timestamp_to_datetime),
is_from_me: row.get::<_, i64>(8)? != 0,
is_read: row.get::<_, i64>(9)? != 0,
is_delivered: row.get::<_, i64>(10)? != 0,
is_sent: row.get::<_, i64>(11)? != 0,
is_emote: row.get::<_, i64>(12)? != 0,
is_audio_message: row.get::<_, i64>(13)? != 0,
cache_has_attachments: row.get::<_, i64>(14)? != 0,
associated_message_guid: row.get(15)?,
associated_message_type: row.get(16)?,
thread_originator_guid: row.get(17)?,
reply_to_guid: row.get(18)?,
is_spam: row.get::<_, i64>(19)? != 0,
})
}

15
crates/lib/src/error.rs Normal file
View File

@@ -0,0 +1,15 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum ChatDbError {
#[error("Database error: {0}")]
Database(#[from] rusqlite::Error),
#[error("Not found: {0}")]
NotFound(String),
#[error("Invalid data: {0}")]
InvalidData(String),
}
pub type Result<T> = std::result::Result<T, ChatDbError>;

30
crates/lib/src/lib.rs Normal file
View File

@@ -0,0 +1,30 @@
//! Data access layer for iMessage chat.db
//!
//! This library provides a read-only interface to query messages from a specific conversation.
//!
//! # Safety
//!
//! All database connections are opened in read-only mode to prevent any
//! accidental modifications to your iMessage database.
//!
//! # Example
//!
//! ```no_run
//! use lib::ChatDb;
//!
//! let db = ChatDb::open("chat.db")?;
//!
//! // Get all messages from January 2024 to now
//! let messages = db.get_our_messages(None, None)?;
//! println!("Found {} messages", messages.len());
//! # Ok::<(), lib::ChatDbError>(())
//! ```
mod error;
mod models;
mod db;
pub mod sync;
pub use error::{ChatDbError, Result};
pub use models::{Message, Chat};
pub use db::ChatDb;

112
crates/lib/src/models.rs Normal file
View File

@@ -0,0 +1,112 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
/// Represents a message in the iMessage database
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Message {
pub rowid: i64,
pub guid: String,
pub text: Option<String>,
pub service: Option<String>,
pub handle_id: i64,
pub date: Option<DateTime<Utc>>,
pub date_read: Option<DateTime<Utc>>,
pub date_delivered: Option<DateTime<Utc>>,
pub is_from_me: bool,
pub is_read: bool,
pub is_delivered: bool,
pub is_sent: bool,
pub is_emote: bool,
pub is_audio_message: bool,
pub cache_has_attachments: bool,
pub associated_message_guid: Option<String>,
pub associated_message_type: i64,
pub thread_originator_guid: Option<String>,
pub reply_to_guid: Option<String>,
pub is_spam: bool,
}
/// Represents a chat/conversation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Chat {
pub rowid: i64,
pub guid: String,
pub chat_identifier: Option<String>,
pub service_name: Option<String>,
pub display_name: Option<String>,
pub group_id: Option<String>,
pub room_name: Option<String>,
pub is_archived: bool,
pub is_filtered: bool,
pub last_read_message_timestamp: Option<DateTime<Utc>>,
}
/// Helper function to convert Apple's Cocoa timestamp (seconds since 2001-01-01) to DateTime
pub fn apple_timestamp_to_datetime(timestamp: i64) -> DateTime<Utc> {
// Apple's Cocoa timestamps are in nanoseconds since 2001-01-01 00:00:00 UTC
// Convert to Unix timestamp (seconds since 1970-01-01 00:00:00 UTC)
const APPLE_EPOCH_OFFSET: i64 = 978307200; // Seconds between 1970-01-01 and 2001-01-01
let seconds = timestamp / 1_000_000_000 + APPLE_EPOCH_OFFSET;
let nanos = (timestamp % 1_000_000_000) as u32;
DateTime::from_timestamp(seconds, nanos).unwrap_or_else(|| DateTime::from_timestamp(0, 0).unwrap())
}
/// Helper function to convert DateTime to Apple's Cocoa timestamp
pub fn datetime_to_apple_timestamp(dt: DateTime<Utc>) -> i64 {
const APPLE_EPOCH_OFFSET: i64 = 978307200;
let unix_timestamp = dt.timestamp();
let nanos = dt.timestamp_subsec_nanos() as i64;
(unix_timestamp - APPLE_EPOCH_OFFSET) * 1_000_000_000 + nanos
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{Datelike, TimeZone, Timelike};
#[test]
fn test_apple_timestamp_to_datetime_zero() {
let dt = apple_timestamp_to_datetime(0);
assert_eq!(dt.year(), 2001);
assert_eq!(dt.month(), 1);
assert_eq!(dt.day(), 1);
assert_eq!(dt.hour(), 0);
assert_eq!(dt.minute(), 0);
assert_eq!(dt.second(), 0);
}
#[test]
fn test_apple_timestamp_to_datetime_known_value() {
let timestamp = 694224000000000000i64;
let dt = apple_timestamp_to_datetime(timestamp);
assert_eq!(dt.year(), 2023);
assert_eq!(dt.month(), 1);
assert_eq!(dt.day(), 1);
}
#[test]
fn test_apple_timestamp_roundtrip() {
let original = 694224000000000000i64;
let dt = apple_timestamp_to_datetime(original);
let converted_back = datetime_to_apple_timestamp(dt);
assert_eq!(original, converted_back);
}
#[test]
fn test_datetime_to_apple_timestamp_epoch() {
let dt = Utc.with_ymd_and_hms(2001, 1, 1, 0, 0, 0).unwrap();
let timestamp = datetime_to_apple_timestamp(dt);
assert_eq!(timestamp, 0);
}
#[test]
fn test_negative_apple_timestamp() {
let timestamp = -31536000000000000i64;
let dt = apple_timestamp_to_datetime(timestamp);
assert_eq!(dt.year(), 2000);
}
}

165
crates/lib/src/sync.rs Normal file
View File

@@ -0,0 +1,165 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::ops::{Deref, DerefMut};
// Re-export the macros
pub use sync_macros::{synced, Synced};
// Re-export common CRDT types from the crdts library
pub use crdts::{
ctx::ReadCtx,
lwwreg::LWWReg,
map::Map,
orswot::Orswot,
CmRDT, CvRDT,
};
pub type NodeId = String;
/// Transparent wrapper for synced values
///
/// This wraps any value with LWW semantics but allows you to use it like a normal value
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncedValue<T: Clone> {
value: T,
timestamp: DateTime<Utc>,
node_id: NodeId,
}
impl<T: Clone> SyncedValue<T> {
pub fn new(value: T, node_id: NodeId) -> Self {
Self {
value,
timestamp: Utc::now(),
node_id,
}
}
pub fn get(&self) -> &T {
&self.value
}
pub fn set(&mut self, value: T, node_id: NodeId) {
self.value = value;
self.timestamp = Utc::now();
self.node_id = node_id;
}
pub fn apply_lww(&mut self, value: T, timestamp: DateTime<Utc>, node_id: NodeId) {
if timestamp > self.timestamp || (timestamp == self.timestamp && node_id > self.node_id) {
self.value = value;
self.timestamp = timestamp;
self.node_id = node_id;
}
}
pub fn merge(&mut self, other: &Self) {
self.apply_lww(other.value.clone(), other.timestamp, other.node_id.clone());
}
}
// Allow transparent access to the inner value
impl<T: Clone> Deref for SyncedValue<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.value
}
}
impl<T: Clone> DerefMut for SyncedValue<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.value
}
}
/// Wrapper for a sync message that goes over gossip
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncMessage<T> {
/// Unique message ID
pub message_id: String,
/// Node that sent this
pub node_id: NodeId,
/// When it was sent
pub timestamp: DateTime<Utc>,
/// The actual sync operation
pub operation: T,
}
impl<T: Serialize> SyncMessage<T> {
pub fn new(node_id: NodeId, operation: T) -> Self {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let seq = COUNTER.fetch_add(1, Ordering::SeqCst);
Self {
message_id: format!("{}-{}-{}", node_id, Utc::now().timestamp_millis(), seq),
node_id,
timestamp: Utc::now(),
operation,
}
}
pub fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
Ok(serde_json::to_vec(self)?)
}
}
impl<T: for<'de> Deserialize<'de>> SyncMessage<T> {
pub fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self> {
Ok(serde_json::from_slice(bytes)?)
}
}
/// Helper trait for types that can be synced
pub trait Syncable: Sized {
type Operation: Serialize + for<'de> Deserialize<'de> + Clone;
/// Apply a sync operation to this value
fn apply_sync_op(&mut self, op: &Self::Operation);
/// Get the node ID for this instance
fn node_id(&self) -> &NodeId;
/// Create a sync message for an operation
fn create_sync_message(&self, op: Self::Operation) -> SyncMessage<Self::Operation> {
SyncMessage::new(self.node_id().clone(), op)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_synced_value() {
let mut val = SyncedValue::new(42, "node1".to_string());
assert_eq!(*val.get(), 42);
val.set(100, "node1".to_string());
assert_eq!(*val.get(), 100);
// Test LWW semantics
let old_time = Utc::now() - chrono::Duration::seconds(10);
val.apply_lww(50, old_time, "node2".to_string());
assert_eq!(*val.get(), 100); // Should not update with older timestamp
}
#[test]
fn test_sync_message() {
#[derive(Debug, Clone, Serialize, Deserialize)]
struct TestOp {
value: i32,
}
let op = TestOp { value: 42 };
let msg = SyncMessage::new("node1".to_string(), op);
let bytes = msg.to_bytes().unwrap();
let decoded = SyncMessage::<TestOp>::from_bytes(&bytes).unwrap();
assert_eq!(decoded.node_id, "node1");
assert_eq!(decoded.operation.value, 42);
}
}

View File

@@ -0,0 +1,98 @@
use lib::{ChatDb, Result};
use chrono::Datelike;
/// Test that we can get messages from the Dutch phone number conversation
#[test]
fn test_get_our_messages_default_range() -> Result<()> {
let db = ChatDb::open("chat.db")?;
// Get messages from January 2024 to now (default)
let messages = db.get_our_messages(None, None)?;
println!("Found {} messages from January 2024 to now", messages.len());
// Verify we got some messages
assert!(messages.len() > 0, "Should find messages in the conversation");
// Verify messages are in chronological order (ASC)
for i in 1..messages.len().min(10) {
if let (Some(prev_date), Some(curr_date)) = (messages[i-1].date, messages[i].date) {
assert!(
prev_date <= curr_date,
"Messages should be in ascending date order"
);
}
}
// Verify all messages are from 2024 or later
for msg in messages.iter().take(10) {
if let Some(date) = msg.date {
assert!(date.year() >= 2024, "Messages should be from 2024 or later");
println!("Message date: {}, from_me: {}, text: {:?}",
date, msg.is_from_me, msg.text.as_ref().map(|s| &s[..s.len().min(50)]));
}
}
Ok(())
}
/// Test that we can get messages with a custom date range
#[test]
fn test_get_our_messages_custom_range() -> Result<()> {
use chrono::{TimeZone, Utc};
let db = ChatDb::open("chat.db")?;
// Get messages from March 2024 to June 2024
let start = Utc.with_ymd_and_hms(2024, 3, 1, 0, 0, 0).unwrap();
let end = Utc.with_ymd_and_hms(2024, 6, 1, 0, 0, 0).unwrap();
let messages = db.get_our_messages(Some(start), Some(end))?;
println!("Found {} messages from March to June 2024", messages.len());
// Verify all messages are within the date range
for msg in &messages {
if let Some(date) = msg.date {
assert!(
date >= start && date <= end,
"Message date {} should be between {} and {}",
date, start, end
);
}
}
Ok(())
}
/// Test displaying a summary of the conversation
#[test]
fn test_conversation_summary() -> Result<()> {
let db = ChatDb::open("chat.db")?;
let messages = db.get_our_messages(None, None)?;
println!("\n=== Conversation Summary ===");
println!("Total messages: {}", messages.len());
let from_me = messages.iter().filter(|m| m.is_from_me).count();
let from_them = messages.len() - from_me;
println!("From me: {}", from_me);
println!("From them: {}", from_them);
// Show first few messages
println!("\nFirst 5 messages:");
for (i, msg) in messages.iter().take(5).enumerate() {
if let Some(date) = msg.date {
let sender = if msg.is_from_me { "Me" } else { "Them" };
let text = msg.text.as_ref()
.map(|t| if t.len() > 60 { format!("{}...", &t[..60]) } else { t.clone() })
.unwrap_or_else(|| "[No text]".to_string());
println!("{}. {} ({}): {}", i + 1, date.format("%Y-%m-%d %H:%M"), sender, text);
}
}
Ok(())
}

View File

@@ -0,0 +1,157 @@
use lib::sync::{synced, SyncMessage, Syncable};
use iroh::{Endpoint, protocol::{Router, ProtocolHandler, AcceptError}};
use anyhow::Result;
use std::sync::Arc;
use tokio::sync::Mutex;
/// Test configuration that can be synced
#[synced]
struct TestConfig {
value: i32,
name: String,
#[sync(skip)]
node_id: String,
}
/// ALPN identifier for our sync protocol
const SYNC_ALPN: &[u8] = b"/lonni/sync/1";
/// Protocol handler for receiving sync messages
#[derive(Debug, Clone)]
struct SyncProtocol {
config: Arc<Mutex<TestConfig>>,
}
impl ProtocolHandler for SyncProtocol {
async fn accept(&self, connection: iroh::endpoint::Connection) -> Result<(), AcceptError> {
println!("Accepting connection from: {}", connection.remote_id());
// Accept the bidirectional stream
let (mut send, mut recv) = connection.accept_bi().await
.map_err(AcceptError::from_err)?;
println!("Stream accepted, reading message...");
// Read the sync message
let bytes = recv.read_to_end(1024 * 1024).await
.map_err(AcceptError::from_err)?;
println!("Received {} bytes", bytes.len());
// Deserialize and apply
let msg = SyncMessage::<TestConfigOp>::from_bytes(&bytes)
.map_err(|e| AcceptError::from_err(std::io::Error::new(std::io::ErrorKind::InvalidData, e)))?;
println!("Applying operation from node: {}", msg.node_id);
let mut config = self.config.lock().await;
config.apply_op(&msg.operation);
println!("Operation applied successfully");
// Close the stream
send.finish()
.map_err(AcceptError::from_err)?;
Ok(())
}
}
#[tokio::test(flavor = "multi_thread")]
async fn test_sync_between_two_nodes() -> Result<()> {
println!("\n=== Testing Sync Between Two Nodes ===\n");
// Create two endpoints
let node1 = Endpoint::builder().bind().await?;
let node2 = Endpoint::builder().bind().await?;
let node1_addr = node1.addr();
let node2_addr = node2.addr();
let node1_id = node1_addr.id.to_string();
let node2_id = node2_addr.id.to_string();
println!("Node 1: {}", node1_id);
println!("Node 2: {}", node2_id);
// Create synced configs on both nodes
let mut config1 = TestConfig::new(
42,
"initial".to_string(),
node1_id.clone(),
);
let config2 = TestConfig::new(
42,
"initial".to_string(),
node2_id.clone(),
);
let config2_shared = Arc::new(Mutex::new(config2));
println!("\nInitial state:");
println!(" Node 1: value={}, name={}", config1.value(), config1.name());
{
let config2 = config2_shared.lock().await;
println!(" Node 2: value={}, name={}", config2.value(), config2.name());
}
// Set up router on node2 to accept incoming connections
println!("\nSetting up node2 router...");
let protocol = SyncProtocol {
config: config2_shared.clone(),
};
let router = Router::builder(node2)
.accept(SYNC_ALPN, protocol)
.spawn();
router.endpoint().online().await;
println!("✓ Node2 router ready");
// Node 1 changes the value
println!("\nNode 1 changing value to 100...");
let op = config1.set_value(100);
// Serialize the operation
let sync_msg = SyncMessage::new(node1_id.clone(), op);
let bytes = sync_msg.to_bytes()?;
println!("Serialized to {} bytes", bytes.len());
// Establish QUIC connection from node1 to node2
println!("\nEstablishing QUIC connection...");
let conn = node1.connect(node2_addr.clone(), SYNC_ALPN).await?;
println!("✓ Connection established");
// Open a bidirectional stream
let (mut send, _recv) = conn.open_bi().await?;
// Send the sync message
println!("Sending sync message...");
send.write_all(&bytes).await?;
send.finish()?;
println!("✓ Message sent");
// Wait a bit for the message to be processed
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
// Verify both configs have the same value
println!("\nFinal state:");
println!(" Node 1: value={}, name={}", config1.value(), config1.name());
{
let config2 = config2_shared.lock().await;
println!(" Node 2: value={}, name={}", config2.value(), config2.name());
assert_eq!(*config1.value(), 100);
assert_eq!(*config2.value(), 100);
assert_eq!(config1.name(), "initial");
assert_eq!(config2.name(), "initial");
}
println!("\n✓ Sync successful!");
// Cleanup
router.shutdown().await?;
node1.close().await;
Ok(())
}