Architecture

// crates/screenpipe-engine/src/paired_capture.rs

pub async fn paired_capture(
    monitor_id: u32,
    trigger: CaptureTrigger,
) -> Result<PairedCaptureResult> {
    // 1. Take screenshot
    let image = capture_monitor_image(monitor_id).await?;
    
    // 2. Get focused window
    let windows = capture_windows().await?;
    
    // 3. Try accessibility first
    let ax_result = walk_focused_window(&windows[0])
        .timeout(Duration::from_millis(200))
        .await?;
    
    // 4. Fallback to OCR if accessibility is empty
    let (text, text_source) = if ax_result.text_content.is_empty() {
        let ocr_text = process_ocr_task(&image).await?;
        (ocr_text, "ocr")
    } else {
        (ax_result.text_content, "accessibility")
    };
    
    // 5. Encode and write JPEG
    let jpeg_path = write_snapshot(&image, monitor_id).await?;
    
    // 6. Insert frame + text into DB
    db.insert_snapshot_frame(jpeg_path, text, trigger, text_source).await?;
    
    Ok(PairedCaptureResult { ... })
}

// crates/screenpipe-screen/src/apple.rs

pub async fn walk_focused_window(window: &Window) -> Result<AccessibilityResult> {
    let ax_element = AXUIElementCreateApplication(window.pid);
    let mut text_content = String::new();
    
    // Walk the accessibility tree with 200ms timeout
    walk_tree_recursive(ax_element, &mut text_content, 0, 200)?;
    
    Ok(AccessibilityResult {
        text_content,
        window_title: window.title.clone(),
        app_name: window.app_name.clone(),
    })
}

// crates/screenpipe-audio/src/lib.rs

pub async fn transcribe_audio_chunk(
    audio_chunk: &[f32],
    device: &str,
) -> Result<Transcription> {
    // 1. Preprocess audio (normalize, denoise)
    let processed = preprocess_audio(audio_chunk)?;
    
    // 2. Run Whisper
    let whisper_output = whisper_model.transcribe(processed).await?;
    
    // 3. Parse speaker diarization
    let segments = parse_speaker_segments(&whisper_output)?;
    
    // 4. Insert into DB
    db.insert_audio_transcription(device, segments).await?;
    
    Ok(Transcription { segments })
}

CREATE TABLE frames (
    id INTEGER PRIMARY KEY,
    timestamp INTEGER NOT NULL,
    device_name TEXT NOT NULL,
    app_name TEXT,
    window_name TEXT,
    snapshot_path TEXT,              -- Path to JPEG file
    accessibility_text TEXT,          -- Text from accessibility tree
    capture_trigger TEXT,             -- 'app_switch', 'click', 'idle', etc.
    text_source TEXT DEFAULT 'ocr',  -- 'accessibility' or 'ocr'
    video_chunk_id INTEGER,           -- Legacy: for old video-based frames
    offset_index INTEGER              -- Legacy: frame offset in video
);

CREATE INDEX idx_frames_ts_device ON frames(timestamp, device_name);
CREATE VIRTUAL TABLE accessibility_text_fts USING fts5(content=frames, accessibility_text);

# Search all content (OCR + audio + accessibility)
GET /search?q=meeting+notes&content_type=all&limit=10

# Search only audio transcriptions
GET /search?q=budget+discussion&content_type=audio&limit=10

# Search by time range
GET /search?start_time=2026-03-08T09:00:00Z&end_time=2026-03-08T17:00:00Z

# Filter by app
GET /search?q=project&app_name=Slack

import { ScreenpipeClient } from "@screenpipe/js";

const client = new ScreenpipeClient();

// Search last 5 minutes
const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString();

const results = await client.search({
  startTime: fiveMinutesAgo,
  limit: 10,
  contentType: "all", // "ocr", "audio", "input", "accessibility", "all"
});

console.log(`Found ${results.pagination.total} items`);

for (const item of results.data) {
  console.log(`Type: ${item.type}`);
  console.log(`Timestamp: ${item.content.timestamp}`);
  
  if (item.type === "OCR") {
    console.log(`Text: ${item.content.text}`);
  } else if (item.type === "Audio") {
    console.log(`Transcript: ${item.content.transcription}`);
  }
}

---
name: obsidian-sync
schedule: "0 */2 * * *"  # Every 2 hours
allow-apps: ["Chrome", "Slack", "VS Code"]
deny-windows: ["*password*", "*credit card*"]
allow-content-types: ["ocr", "accessibility"]
time-range: "09:00-18:00"
days: "Mon,Tue,Wed,Thu,Fri"
---

# Obsidian Sync Pipe

You are an AI assistant that syncs screen activity to Obsidian.

## Task

1. Query screenpipe for activity in the last 2 hours
2. Filter for work-related apps (Chrome, Slack, VS Code)
3. Extract key events (meetings, code commits, Slack messages)
4. Write a daily log to `~/Obsidian/Daily Notes/YYYY-MM-DD.md`

## API Access

You have access to the screenpipe API at `http://localhost:3030`.

Example query:
```bash
curl "http://localhost:3030/search?content_type=all&limit=50"