Shifty/Mastermind
Shifty/Mastermind
1
1
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: implement Whisper worker for isolated audio transcription

Browse Source
This commit is contained in:
Илья Глазунов 2026-02-16 11:38:26 +03:00
parent 1b74968006
commit 684b61755c
3 changed files with 1027 additions and 570 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
src/index.js
View File

@ -1,11 +1,43 @@
if (require('electron-squirrel-startup')) { if (require("electron-squirrel-startup")) {
process.exit(0); process.exit(0);
} }
const { app, BrowserWindow, shell, ipcMain } = require('electron'); // ── Global crash handlers to prevent silent process termination ──
const { createWindow, updateGlobalShortcuts } = require('./utils/window'); process.on("uncaughtException", (error) => {
const { setupGeminiIpcHandlers, stopMacOSAudioCapture, sendToRenderer } = require('./utils/gemini'); console.error("[FATAL] Uncaught exception:", error);
const storage = require('./storage'); try {
const { sendToRenderer } = require("./utils/gemini");
sendToRenderer(
"update-status",
"Fatal error: " + (error?.message || "unknown"),
);
} catch (_) {
// sendToRenderer may not be available yet
}
});
process.on("unhandledRejection", (reason) => {
console.error("[FATAL] Unhandled promise rejection:", reason);
try {
const { sendToRenderer } = require("./utils/gemini");
sendToRenderer(
"update-status",
"Unhandled error: " +
(reason instanceof Error ? reason.message : String(reason)),
);
} catch (_) {
// sendToRenderer may not be available yet
}
});
const { app, BrowserWindow, shell, ipcMain } = require("electron");
const { createWindow, updateGlobalShortcuts } = require("./utils/window");
const {
setupGeminiIpcHandlers,
stopMacOSAudioCapture,
sendToRenderer,
} = require("./utils/gemini");
const storage = require("./storage");
const geminiSessionRef = { current: null }; const geminiSessionRef = { current: null };
let mainWindow = null; let mainWindow = null;
@ -20,9 +52,9 @@ app.whenReady().then(async () => {
storage.initializeStorage(); storage.initializeStorage();
// Trigger screen recording permission prompt on macOS if not already granted // Trigger screen recording permission prompt on macOS if not already granted
if (process.platform === 'darwin') { if (process.platform === "darwin") {
const { desktopCapturer } = require('electron'); const { desktopCapturer } = require("electron");
desktopCapturer.getSources({ types: ['screen'] }).catch(() => {}); desktopCapturer.getSources({ types: ["screen"] }).catch(() => {});
} }
createMainWindow(); createMainWindow();
@ -31,18 +63,18 @@ app.whenReady().then(async () => {
setupGeneralIpcHandlers(); setupGeneralIpcHandlers();
}); });
app.on('window-all-closed', () => { app.on("window-all-closed", () => {
stopMacOSAudioCapture(); stopMacOSAudioCapture();
if (process.platform !== 'darwin') { if (process.platform !== "darwin") {
app.quit(); app.quit();
} }
}); });
app.on('before-quit', () => { app.on("before-quit", () => {
stopMacOSAudioCapture(); stopMacOSAudioCapture();
}); });
app.on('activate', () => { app.on("activate", () => {
if (BrowserWindow.getAllWindows().length === 0) { if (BrowserWindow.getAllWindows().length === 0) {
createMainWindow(); createMainWindow();
} }
@ -50,250 +82,255 @@ app.on('activate', () => {
function setupStorageIpcHandlers() { function setupStorageIpcHandlers() {
// ============ CONFIG ============ // ============ CONFIG ============
ipcMain.handle('storage:get-config', async () => { ipcMain.handle("storage:get-config", async () => {
try { try {
return { success: true, data: storage.getConfig() }; return { success: true, data: storage.getConfig() };
} catch (error) { } catch (error) {
console.error('Error getting config:', error); console.error("Error getting config:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:set-config', async (event, config) => { ipcMain.handle("storage:set-config", async (event, config) => {
try { try {
storage.setConfig(config); storage.setConfig(config);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error setting config:', error); console.error("Error setting config:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:update-config', async (event, key, value) => { ipcMain.handle("storage:update-config", async (event, key, value) => {
try { try {
storage.updateConfig(key, value); storage.updateConfig(key, value);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error updating config:', error); console.error("Error updating config:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
// ============ CREDENTIALS ============ // ============ CREDENTIALS ============
ipcMain.handle('storage:get-credentials', async () => { ipcMain.handle("storage:get-credentials", async () => {
try { try {
return { success: true, data: storage.getCredentials() }; return { success: true, data: storage.getCredentials() };
} catch (error) { } catch (error) {
console.error('Error getting credentials:', error); console.error("Error getting credentials:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:set-credentials', async (event, credentials) => { ipcMain.handle("storage:set-credentials", async (event, credentials) => {
try { try {
storage.setCredentials(credentials); storage.setCredentials(credentials);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error setting credentials:', error); console.error("Error setting credentials:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:get-api-key', async () => { ipcMain.handle("storage:get-api-key", async () => {
try { try {
return { success: true, data: storage.getApiKey() }; return { success: true, data: storage.getApiKey() };
} catch (error) { } catch (error) {
console.error('Error getting API key:', error); console.error("Error getting API key:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:set-api-key', async (event, apiKey) => { ipcMain.handle("storage:set-api-key", async (event, apiKey) => {
try { try {
storage.setApiKey(apiKey); storage.setApiKey(apiKey);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error setting API key:', error); console.error("Error setting API key:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:get-groq-api-key', async () => { ipcMain.handle("storage:get-groq-api-key", async () => {
try { try {
return { success: true, data: storage.getGroqApiKey() }; return { success: true, data: storage.getGroqApiKey() };
} catch (error) { } catch (error) {
console.error('Error getting Groq API key:', error); console.error("Error getting Groq API key:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:set-groq-api-key', async (event, groqApiKey) => { ipcMain.handle("storage:set-groq-api-key", async (event, groqApiKey) => {
try { try {
storage.setGroqApiKey(groqApiKey); storage.setGroqApiKey(groqApiKey);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error setting Groq API key:', error); console.error("Error setting Groq API key:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
// ============ PREFERENCES ============ // ============ PREFERENCES ============
ipcMain.handle('storage:get-preferences', async () => { ipcMain.handle("storage:get-preferences", async () => {
try { try {
return { success: true, data: storage.getPreferences() }; return { success: true, data: storage.getPreferences() };
} catch (error) { } catch (error) {
console.error('Error getting preferences:', error); console.error("Error getting preferences:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:set-preferences', async (event, preferences) => { ipcMain.handle("storage:set-preferences", async (event, preferences) => {
try { try {
storage.setPreferences(preferences); storage.setPreferences(preferences);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error setting preferences:', error); console.error("Error setting preferences:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:update-preference', async (event, key, value) => { ipcMain.handle("storage:update-preference", async (event, key, value) => {
try { try {
storage.updatePreference(key, value); storage.updatePreference(key, value);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error updating preference:', error); console.error("Error updating preference:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
// ============ KEYBINDS ============ // ============ KEYBINDS ============
ipcMain.handle('storage:get-keybinds', async () => { ipcMain.handle("storage:get-keybinds", async () => {
try { try {
return { success: true, data: storage.getKeybinds() }; return { success: true, data: storage.getKeybinds() };
} catch (error) { } catch (error) {
console.error('Error getting keybinds:', error); console.error("Error getting keybinds:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:set-keybinds', async (event, keybinds) => { ipcMain.handle("storage:set-keybinds", async (event, keybinds) => {
try { try {
storage.setKeybinds(keybinds); storage.setKeybinds(keybinds);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error setting keybinds:', error); console.error("Error setting keybinds:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
// ============ HISTORY ============ // ============ HISTORY ============
ipcMain.handle('storage:get-all-sessions', async () => { ipcMain.handle("storage:get-all-sessions", async () => {
try { try {
return { success: true, data: storage.getAllSessions() }; return { success: true, data: storage.getAllSessions() };
} catch (error) { } catch (error) {
console.error('Error getting sessions:', error); console.error("Error getting sessions:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:get-session', async (event, sessionId) => { ipcMain.handle("storage:get-session", async (event, sessionId) => {
try { try {
return { success: true, data: storage.getSession(sessionId) }; return { success: true, data: storage.getSession(sessionId) };
} catch (error) { } catch (error) {
console.error('Error getting session:', error); console.error("Error getting session:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:save-session', async (event, sessionId, data) => { ipcMain.handle("storage:save-session", async (event, sessionId, data) => {
try { try {
storage.saveSession(sessionId, data); storage.saveSession(sessionId, data);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error saving session:', error); console.error("Error saving session:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:delete-session', async (event, sessionId) => { ipcMain.handle("storage:delete-session", async (event, sessionId) => {
try { try {
storage.deleteSession(sessionId); storage.deleteSession(sessionId);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error deleting session:', error); console.error("Error deleting session:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('storage:delete-all-sessions', async () => { ipcMain.handle("storage:delete-all-sessions", async () => {
try { try {
storage.deleteAllSessions(); storage.deleteAllSessions();
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error deleting all sessions:', error); console.error("Error deleting all sessions:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
// ============ LIMITS ============ // ============ LIMITS ============
ipcMain.handle('storage:get-today-limits', async () => { ipcMain.handle("storage:get-today-limits", async () => {
try { try {
return { success: true, data: storage.getTodayLimits() }; return { success: true, data: storage.getTodayLimits() };
} catch (error) { } catch (error) {
console.error('Error getting today limits:', error); console.error("Error getting today limits:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
// ============ CLEAR ALL ============ // ============ CLEAR ALL ============
ipcMain.handle('storage:clear-all', async () => { ipcMain.handle("storage:clear-all", async () => {
try { try {
storage.clearAllData(); storage.clearAllData();
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error clearing all data:', error); console.error("Error clearing all data:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
} }
function setupGeneralIpcHandlers() { function setupGeneralIpcHandlers() {
ipcMain.handle('get-app-version', async () => { ipcMain.handle("get-app-version", async () => {
return app.getVersion(); return app.getVersion();
}); });
ipcMain.handle('quit-application', async event => { ipcMain.handle("quit-application", async (event) => {
try { try {
stopMacOSAudioCapture(); stopMacOSAudioCapture();
app.quit(); app.quit();
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error quitting application:', error); console.error("Error quitting application:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.handle('open-external', async (event, url) => { ipcMain.handle("open-external", async (event, url) => {
try { try {
await shell.openExternal(url); await shell.openExternal(url);
return { success: true }; return { success: true };
} catch (error) { } catch (error) {
console.error('Error opening external URL:', error); console.error("Error opening external URL:", error);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
}); });
ipcMain.on('update-keybinds', (event, newKeybinds) => { ipcMain.on("update-keybinds", (event, newKeybinds) => {
if (mainWindow) { if (mainWindow) {
// Also save to storage // Also save to storage
storage.setKeybinds(newKeybinds); storage.setKeybinds(newKeybinds);
updateGlobalShortcuts(newKeybinds, mainWindow, sendToRenderer, geminiSessionRef); updateGlobalShortcuts(
newKeybinds,
mainWindow,
sendToRenderer,
geminiSessionRef,
);
} }
}); });
// Debug logging from renderer // Debug logging from renderer
ipcMain.on('log-message', (event, msg) => { ipcMain.on("log-message", (event, msg) => {
console.log(msg); console.log(msg);
}); });
} }

504
src/utils/localai.js
View File

@ -1,17 +1,27 @@
const { Ollama } = require('ollama'); const { Ollama } = require("ollama");
const { getSystemPrompt } = require('./prompts'); const { getSystemPrompt } = require("./prompts");
const { sendToRenderer, initializeNewSession, saveConversationTurn } = require('./gemini'); const {
sendToRenderer,
initializeNewSession,
saveConversationTurn,
} = require("./gemini");
const { fork } = require("child_process");
const path = require("path");
// ── State ── // ── State ──
let ollamaClient = null; let ollamaClient = null;
let ollamaModel = null; let ollamaModel = null;
let whisperPipeline = null; let whisperWorker = null;
let isWhisperLoading = false; let isWhisperLoading = false;
let whisperReady = false;
let localConversationHistory = []; let localConversationHistory = [];
let currentSystemPrompt = null; let currentSystemPrompt = null;
let isLocalActive = false; let isLocalActive = false;
// Pending transcription callback (one at a time)
let pendingTranscribe = null;
// VAD state // VAD state
let isSpeaking = false; let isSpeaking = false;
let speechBuffers = []; let speechBuffers = [];
@ -20,13 +30,32 @@ let speechFrameCount = 0;
// VAD configuration // VAD configuration
const VAD_MODES = { const VAD_MODES = {
NORMAL: { energyThreshold: 0.01, speechFramesRequired: 3, silenceFramesRequired: 30 }, NORMAL: {
LOW_BITRATE: { energyThreshold: 0.008, speechFramesRequired: 4, silenceFramesRequired: 35 }, energyThreshold: 0.01,
AGGRESSIVE: { energyThreshold: 0.015, speechFramesRequired: 2, silenceFramesRequired: 20 }, speechFramesRequired: 3,
VERY_AGGRESSIVE: { energyThreshold: 0.02, speechFramesRequired: 2, silenceFramesRequired: 15 }, silenceFramesRequired: 30,
},
LOW_BITRATE: {
energyThreshold: 0.008,
speechFramesRequired: 4,
silenceFramesRequired: 35,
},
AGGRESSIVE: {
energyThreshold: 0.015,
speechFramesRequired: 2,
silenceFramesRequired: 20,
},
VERY_AGGRESSIVE: {
energyThreshold: 0.02,
speechFramesRequired: 2,
silenceFramesRequired: 15,
},
}; };
let vadConfig = VAD_MODES.VERY_AGGRESSIVE; let vadConfig = VAD_MODES.VERY_AGGRESSIVE;
// Maximum speech buffer size: ~30 seconds at 16kHz, 16-bit mono
const MAX_SPEECH_BUFFER_BYTES = 16000 * 2 * 30; // 960,000 bytes
// Audio resampling buffer // Audio resampling buffer
let resampleRemainder = Buffer.alloc(0); let resampleRemainder = Buffer.alloc(0);
@ -47,15 +76,24 @@ function resample24kTo16k(inputBuffer) {
const frac = srcPos - srcIndex; const frac = srcPos - srcIndex;
const s0 = combined.readInt16LE(srcIndex * 2); const s0 = combined.readInt16LE(srcIndex * 2);
const s1 = srcIndex + 1 < inputSamples ? combined.readInt16LE((srcIndex + 1) * 2) : s0; const s1 =
srcIndex + 1 < inputSamples
? combined.readInt16LE((srcIndex + 1) * 2)
: s0;
const interpolated = Math.round(s0 + frac * (s1 - s0)); const interpolated = Math.round(s0 + frac * (s1 - s0));
outputBuffer.writeInt16LE(Math.max(-32768, Math.min(32767, interpolated)), i * 2); outputBuffer.writeInt16LE(
Math.max(-32768, Math.min(32767, interpolated)),
i * 2,
);
} }
// Store remainder for next call // Store remainder for next call
const consumedInputSamples = Math.ceil((outputSamples * 3) / 2); const consumedInputSamples = Math.ceil((outputSamples * 3) / 2);
const remainderStart = consumedInputSamples * 2; const remainderStart = consumedInputSamples * 2;
resampleRemainder = remainderStart < combined.length ? combined.slice(remainderStart) : Buffer.alloc(0); resampleRemainder =
remainderStart < combined.length
? combined.slice(remainderStart)
: Buffer.alloc(0);
return outputBuffer; return outputBuffer;
} }
@ -84,8 +122,8 @@ function processVAD(pcm16kBuffer) {
if (!isSpeaking && speechFrameCount >= vadConfig.speechFramesRequired) { if (!isSpeaking && speechFrameCount >= vadConfig.speechFramesRequired) {
isSpeaking = true; isSpeaking = true;
speechBuffers = []; speechBuffers = [];
console.log('[LocalAI] Speech started (RMS:', rms.toFixed(4), ')'); console.log("[LocalAI] Speech started (RMS:", rms.toFixed(4), ")");
sendToRenderer('update-status', 'Listening... (speech detected)'); sendToRenderer("update-status", "Listening... (speech detected)");
} }
} else { } else {
silenceFrameCount++; silenceFrameCount++;
@ -93,13 +131,23 @@ function processVAD(pcm16kBuffer) {
if (isSpeaking && silenceFrameCount >= vadConfig.silenceFramesRequired) { if (isSpeaking && silenceFrameCount >= vadConfig.silenceFramesRequired) {
isSpeaking = false; isSpeaking = false;
console.log('[LocalAI] Speech ended, accumulated', speechBuffers.length, 'chunks'); console.log(
sendToRenderer('update-status', 'Transcribing...'); "[LocalAI] Speech ended, accumulated",
speechBuffers.length,
"chunks",
);
sendToRenderer("update-status", "Transcribing...");
// Trigger transcription with accumulated audio // Trigger transcription with accumulated audio
const audioData = Buffer.concat(speechBuffers); const audioData = Buffer.concat(speechBuffers);
speechBuffers = []; speechBuffers = [];
handleSpeechEnd(audioData); handleSpeechEnd(audioData).catch((err) => {
console.error("[LocalAI] handleSpeechEnd crashed:", err);
sendToRenderer(
"update-status",
"Transcription error: " + (err?.message || "unknown"),
);
});
return; return;
} }
} }
@ -107,76 +155,252 @@ function processVAD(pcm16kBuffer) {
// Accumulate audio during speech // Accumulate audio during speech
if (isSpeaking) { if (isSpeaking) {
speechBuffers.push(Buffer.from(pcm16kBuffer)); speechBuffers.push(Buffer.from(pcm16kBuffer));
// Cap buffer at ~30 seconds to prevent OOM and ONNX tensor overflow
const totalBytes = speechBuffers.reduce((sum, b) => sum + b.length, 0);
if (totalBytes >= MAX_SPEECH_BUFFER_BYTES) {
isSpeaking = false;
console.log(
"[LocalAI] Speech buffer limit reached (" +
totalBytes +
" bytes), forcing transcription",
);
sendToRenderer("update-status", "Transcribing (max length reached)...");
const audioData = Buffer.concat(speechBuffers);
speechBuffers = [];
silenceFrameCount = 0;
speechFrameCount = 0;
handleSpeechEnd(audioData).catch((err) => {
console.error("[LocalAI] handleSpeechEnd crashed:", err);
sendToRenderer(
"update-status",
"Transcription error: " + (err?.message || "unknown"),
);
});
}
} }
} }
// ── Whisper Transcription ── // ── Whisper Worker (isolated child process) ──
function spawnWhisperWorker() {
if (whisperWorker) return;
const workerPath = path.join(__dirname, "whisperWorker.js");
console.log("[LocalAI] Spawning Whisper worker:", workerPath);
whisperWorker = fork(workerPath, [], {
stdio: ["pipe", "pipe", "pipe", "ipc"],
// ELECTRON_RUN_AS_NODE makes the Electron binary behave as plain Node.js,
// which is required for child_process.fork() in packaged Electron apps.
env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" },
});
whisperWorker.stdout.on("data", (data) => {
console.log("[WhisperWorker stdout]", data.toString().trim());
});
whisperWorker.stderr.on("data", (data) => {
console.error("[WhisperWorker stderr]", data.toString().trim());
});
whisperWorker.on("message", (msg) => {
switch (msg.type) {
case "ready":
console.log("[LocalAI] Whisper worker ready");
break;
case "load-result":
handleWorkerLoadResult(msg);
break;
case "transcribe-result":
handleWorkerTranscribeResult(msg);
break;
case "status":
sendToRenderer("update-status", msg.message);
break;
}
});
whisperWorker.on("exit", (code, signal) => {
console.error(
"[LocalAI] Whisper worker exited — code:",
code,
"signal:",
signal,
);
whisperWorker = null;
whisperReady = false;
// Reject any pending transcription
if (pendingTranscribe) {
pendingTranscribe.reject(
new Error(
"Whisper worker crashed (code: " + code + ", signal: " + signal + ")",
),
);
pendingTranscribe = null;
}
// If session is still active, inform the user and respawn
if (isLocalActive) {
sendToRenderer(
"update-status",
"Whisper crashed (signal: " +
(signal || code) +
"). Respawning worker...",
);
setTimeout(() => {
if (isLocalActive) {
respawnWhisperWorker();
}
}, 2000);
}
});
whisperWorker.on("error", (err) => {
console.error("[LocalAI] Whisper worker error:", err);
whisperWorker = null;
whisperReady = false;
});
}
let pendingLoad = null;
function handleWorkerLoadResult(msg) {
if (msg.success) {
console.log("[LocalAI] Whisper model loaded successfully (in worker)");
whisperReady = true;
sendToRenderer("whisper-downloading", false);
isWhisperLoading = false;
if (pendingLoad) {
pendingLoad.resolve(true);
pendingLoad = null;
}
} else {
console.error("[LocalAI] Whisper worker failed to load model:", msg.error);
sendToRenderer("whisper-downloading", false);
sendToRenderer(
"update-status",
"Failed to load Whisper model: " + msg.error,
);
isWhisperLoading = false;
if (pendingLoad) {
pendingLoad.resolve(false);
pendingLoad = null;
}
}
}
function handleWorkerTranscribeResult(msg) {
if (!pendingTranscribe) return;
if (msg.success) {
console.log("[LocalAI] Transcription:", msg.text);
pendingTranscribe.resolve(msg.text || null);
} else {
console.error("[LocalAI] Worker transcription error:", msg.error);
pendingTranscribe.resolve(null);
}
pendingTranscribe = null;
}
function respawnWhisperWorker() {
killWhisperWorker();
spawnWhisperWorker();
const { app } = require("electron");
const cacheDir = path.join(app.getPath("userData"), "whisper-models");
const modelName =
require("../storage").getPreferences().whisperModel ||
"Xenova/whisper-small";
sendToRenderer("whisper-downloading", true);
isWhisperLoading = true;
whisperWorker.send({ type: "load", modelName, cacheDir });
}
function killWhisperWorker() {
if (whisperWorker) {
try {
whisperWorker.removeAllListeners();
whisperWorker.kill();
} catch (_) {
// Already dead
}
whisperWorker = null;
whisperReady = false;
}
}
async function loadWhisperPipeline(modelName) { async function loadWhisperPipeline(modelName) {
if (whisperPipeline) return whisperPipeline; if (whisperReady) return true;
if (isWhisperLoading) return null; if (isWhisperLoading) return null;
isWhisperLoading = true; isWhisperLoading = true;
console.log('[LocalAI] Loading Whisper model:', modelName); console.log("[LocalAI] Loading Whisper model via worker:", modelName);
sendToRenderer('whisper-downloading', true); sendToRenderer("whisper-downloading", true);
sendToRenderer('update-status', 'Loading Whisper model (first time may take a while)...'); sendToRenderer(
"update-status",
"Loading Whisper model (first time may take a while)...",
);
try { spawnWhisperWorker();
// Dynamic import for ESM module
const { pipeline, env } = await import('@huggingface/transformers'); const { app } = require("electron");
// Cache models outside the asar archive so ONNX runtime can load them const cacheDir = path.join(app.getPath("userData"), "whisper-models");
const { app } = require('electron');
const path = require('path'); return new Promise((resolve) => {
env.cacheDir = path.join(app.getPath('userData'), 'whisper-models'); pendingLoad = { resolve };
whisperPipeline = await pipeline('automatic-speech-recognition', modelName, { whisperWorker.send({ type: "load", modelName, cacheDir });
dtype: 'q8',
device: 'auto',
}); });
console.log('[LocalAI] Whisper model loaded successfully');
sendToRenderer('whisper-downloading', false);
isWhisperLoading = false;
return whisperPipeline;
} catch (error) {
console.error('[LocalAI] Failed to load Whisper model:', error);
sendToRenderer('whisper-downloading', false);
sendToRenderer('update-status', 'Failed to load Whisper model: ' + error.message);
isWhisperLoading = false;
return null;
}
}
function pcm16ToFloat32(pcm16Buffer) {
const samples = pcm16Buffer.length / 2;
const float32 = new Float32Array(samples);
for (let i = 0; i < samples; i++) {
float32[i] = pcm16Buffer.readInt16LE(i * 2) / 32768;
}
return float32;
} }
async function transcribeAudio(pcm16kBuffer) { async function transcribeAudio(pcm16kBuffer) {
if (!whisperPipeline) { if (!whisperReady || !whisperWorker) {
console.error('[LocalAI] Whisper pipeline not loaded'); console.error("[LocalAI] Whisper worker not ready");
return null; return null;
} }
if (!pcm16kBuffer || pcm16kBuffer.length < 2) {
console.error("[LocalAI] Invalid audio buffer:", pcm16kBuffer?.length);
return null;
}
console.log(
"[LocalAI] Starting transcription, audio length:",
pcm16kBuffer.length,
"bytes",
);
// Send audio to worker as base64 (IPC serialization)
const audioBase64 = pcm16kBuffer.toString("base64");
return new Promise((resolve, reject) => {
// Timeout: if worker takes > 60s, assume it's stuck
const timeout = setTimeout(() => {
console.error("[LocalAI] Transcription timed out after 60s");
if (pendingTranscribe) {
pendingTranscribe = null;
resolve(null);
}
}, 60000);
pendingTranscribe = {
resolve: (val) => {
clearTimeout(timeout);
resolve(val);
},
reject: (err) => {
clearTimeout(timeout);
reject(err);
},
};
try { try {
const float32Audio = pcm16ToFloat32(pcm16kBuffer); whisperWorker.send({ type: "transcribe", audioBase64 });
} catch (err) {
// Whisper expects audio at 16kHz which is what we have clearTimeout(timeout);
const result = await whisperPipeline(float32Audio, { pendingTranscribe = null;
sampling_rate: 16000, console.error("[LocalAI] Failed to send to worker:", err);
language: 'en', resolve(null);
task: 'transcribe',
});
const text = result.text?.trim();
console.log('[LocalAI] Transcription:', text);
return text;
} catch (error) {
console.error('[LocalAI] Transcription error:', error);
return null;
} }
});
} }
// ── Speech End Handler ── // ── Speech End Handler ──
@ -186,35 +410,52 @@ async function handleSpeechEnd(audioData) {
// Minimum audio length check (~0.5 seconds at 16kHz, 16-bit) // Minimum audio length check (~0.5 seconds at 16kHz, 16-bit)
if (audioData.length < 16000) { if (audioData.length < 16000) {
console.log('[LocalAI] Audio too short, skipping'); console.log("[LocalAI] Audio too short, skipping");
sendToRenderer('update-status', 'Listening...'); sendToRenderer("update-status", "Listening...");
return; return;
} }
console.log("[LocalAI] Processing audio:", audioData.length, "bytes");
try {
const transcription = await transcribeAudio(audioData); const transcription = await transcribeAudio(audioData);
if (!transcription || transcription.trim() === '' || transcription.trim().length < 2) { if (
console.log('[LocalAI] Empty transcription, skipping'); !transcription ||
sendToRenderer('update-status', 'Listening...'); transcription.trim() === "" ||
transcription.trim().length < 2
) {
console.log("[LocalAI] Empty transcription, skipping");
sendToRenderer("update-status", "Listening...");
return; return;
} }
sendToRenderer('update-status', 'Generating response...'); sendToRenderer("update-status", "Generating response...");
await sendToOllama(transcription); await sendToOllama(transcription);
} catch (error) {
console.error("[LocalAI] handleSpeechEnd error:", error);
sendToRenderer(
"update-status",
"Error: " + (error?.message || "transcription failed"),
);
}
} }
// ── Ollama Chat ── // ── Ollama Chat ──
async function sendToOllama(transcription) { async function sendToOllama(transcription) {
if (!ollamaClient || !ollamaModel) { if (!ollamaClient || !ollamaModel) {
console.error('[LocalAI] Ollama not configured'); console.error("[LocalAI] Ollama not configured");
return; return;
} }
console.log('[LocalAI] Sending to Ollama:', transcription.substring(0, 100) + '...'); console.log(
"[LocalAI] Sending to Ollama:",
transcription.substring(0, 100) + "...",
);
localConversationHistory.push({ localConversationHistory.push({
role: 'user', role: "user",
content: transcription.trim(), content: transcription.trim(),
}); });
@ -225,7 +466,10 @@ async function sendToOllama(transcription) {
try { try {
const messages = [ const messages = [
{ role: 'system', content: currentSystemPrompt || 'You are a helpful assistant.' }, {
role: "system",
content: currentSystemPrompt || "You are a helpful assistant.",
},
...localConversationHistory, ...localConversationHistory,
]; ];
@ -235,41 +479,52 @@ async function sendToOllama(transcription) {
stream: true, stream: true,
}); });
let fullText = ''; let fullText = "";
let isFirst = true; let isFirst = true;
for await (const part of response) { for await (const part of response) {
const token = part.message?.content || ''; const token = part.message?.content || "";
if (token) { if (token) {
fullText += token; fullText += token;
sendToRenderer(isFirst ? 'new-response' : 'update-response', fullText); sendToRenderer(isFirst ? "new-response" : "update-response", fullText);
isFirst = false; isFirst = false;
} }
} }
if (fullText.trim()) { if (fullText.trim()) {
localConversationHistory.push({ localConversationHistory.push({
role: 'assistant', role: "assistant",
content: fullText.trim(), content: fullText.trim(),
}); });
saveConversationTurn(transcription, fullText); saveConversationTurn(transcription, fullText);
} }
console.log('[LocalAI] Ollama response completed'); console.log("[LocalAI] Ollama response completed");
sendToRenderer('update-status', 'Listening...'); sendToRenderer("update-status", "Listening...");
} catch (error) { } catch (error) {
console.error('[LocalAI] Ollama error:', error); console.error("[LocalAI] Ollama error:", error);
sendToRenderer('update-status', 'Ollama error: ' + error.message); sendToRenderer("update-status", "Ollama error: " + error.message);
} }
} }
// ── Public API ── // ── Public API ──
async function initializeLocalSession(ollamaHost, model, whisperModel, profile, customPrompt) { async function initializeLocalSession(
console.log('[LocalAI] Initializing local session:', { ollamaHost, model, whisperModel, profile }); ollamaHost,
model,
whisperModel,
profile,
customPrompt,
) {
console.log("[LocalAI] Initializing local session:", {
ollamaHost,
model,
whisperModel,
profile,
});
sendToRenderer('session-initializing', true); sendToRenderer("session-initializing", true);
try { try {
// Setup system prompt // Setup system prompt
@ -282,18 +537,26 @@ async function initializeLocalSession(ollamaHost, model, whisperModel, profile,
// Test Ollama connection // Test Ollama connection
try { try {
await ollamaClient.list(); await ollamaClient.list();
console.log('[LocalAI] Ollama connection verified'); console.log("[LocalAI] Ollama connection verified");
} catch (error) { } catch (error) {
console.error('[LocalAI] Cannot connect to Ollama at', ollamaHost, ':', error.message); console.error(
sendToRenderer('session-initializing', false); "[LocalAI] Cannot connect to Ollama at",
sendToRenderer('update-status', 'Cannot connect to Ollama at ' + ollamaHost); ollamaHost,
":",
error.message,
);
sendToRenderer("session-initializing", false);
sendToRenderer(
"update-status",
"Cannot connect to Ollama at " + ollamaHost,
);
return false; return false;
} }
// Load Whisper model // Load Whisper model
const pipeline = await loadWhisperPipeline(whisperModel); const pipeline = await loadWhisperPipeline(whisperModel);
if (!pipeline) { if (!pipeline) {
sendToRenderer('session-initializing', false); sendToRenderer("session-initializing", false);
return false; return false;
} }
@ -309,15 +572,15 @@ async function initializeLocalSession(ollamaHost, model, whisperModel, profile,
initializeNewSession(profile, customPrompt); initializeNewSession(profile, customPrompt);
isLocalActive = true; isLocalActive = true;
sendToRenderer('session-initializing', false); sendToRenderer("session-initializing", false);
sendToRenderer('update-status', 'Local AI ready - Listening...'); sendToRenderer("update-status", "Local AI ready - Listening...");
console.log('[LocalAI] Session initialized successfully'); console.log("[LocalAI] Session initialized successfully");
return true; return true;
} catch (error) { } catch (error) {
console.error('[LocalAI] Initialization error:', error); console.error("[LocalAI] Initialization error:", error);
sendToRenderer('session-initializing', false); sendToRenderer("session-initializing", false);
sendToRenderer('update-status', 'Local AI error: ' + error.message); sendToRenderer("update-status", "Local AI error: " + error.message);
return false; return false;
} }
} }
@ -333,7 +596,7 @@ function processLocalAudio(monoChunk24k) {
} }
function closeLocalSession() { function closeLocalSession() {
console.log('[LocalAI] Closing local session'); console.log("[LocalAI] Closing local session");
isLocalActive = false; isLocalActive = false;
isSpeaking = false; isSpeaking = false;
speechBuffers = []; speechBuffers = [];
@ -344,7 +607,8 @@ function closeLocalSession() {
ollamaClient = null; ollamaClient = null;
ollamaModel = null; ollamaModel = null;
currentSystemPrompt = null; currentSystemPrompt = null;
// Note: whisperPipeline is kept loaded to avoid reloading on next session // Note: whisperWorker is kept alive to avoid reloading model on next session
// To fully clean up, call killWhisperWorker()
} }
function isLocalSessionActive() { function isLocalSessionActive() {
@ -355,7 +619,7 @@ function isLocalSessionActive() {
async function sendLocalText(text) { async function sendLocalText(text) {
if (!isLocalActive || !ollamaClient) { if (!isLocalActive || !ollamaClient) {
return { success: false, error: 'No active local session' }; return { success: false, error: "No active local session" };
} }
try { try {
@ -368,28 +632,31 @@ async function sendLocalText(text) {
async function sendLocalImage(base64Data, prompt) { async function sendLocalImage(base64Data, prompt) {
if (!isLocalActive || !ollamaClient) { if (!isLocalActive || !ollamaClient) {
return { success: false, error: 'No active local session' }; return { success: false, error: "No active local session" };
} }
try { try {
console.log('[LocalAI] Sending image to Ollama'); console.log("[LocalAI] Sending image to Ollama");
sendToRenderer('update-status', 'Analyzing image...'); sendToRenderer("update-status", "Analyzing image...");
const userMessage = { const userMessage = {
role: 'user', role: "user",
content: prompt, content: prompt,
images: [base64Data], images: [base64Data],
}; };
// Store text-only version in history // Store text-only version in history
localConversationHistory.push({ role: 'user', content: prompt }); localConversationHistory.push({ role: "user", content: prompt });
if (localConversationHistory.length > 20) { if (localConversationHistory.length > 20) {
localConversationHistory = localConversationHistory.slice(-20); localConversationHistory = localConversationHistory.slice(-20);
} }
const messages = [ const messages = [
{ role: 'system', content: currentSystemPrompt || 'You are a helpful assistant.' }, {
role: "system",
content: currentSystemPrompt || "You are a helpful assistant.",
},
...localConversationHistory.slice(0, -1), ...localConversationHistory.slice(0, -1),
userMessage, userMessage,
]; ];
@ -400,29 +667,32 @@ async function sendLocalImage(base64Data, prompt) {
stream: true, stream: true,
}); });
let fullText = ''; let fullText = "";
let isFirst = true; let isFirst = true;
for await (const part of response) { for await (const part of response) {
const token = part.message?.content || ''; const token = part.message?.content || "";
if (token) { if (token) {
fullText += token; fullText += token;
sendToRenderer(isFirst ? 'new-response' : 'update-response', fullText); sendToRenderer(isFirst ? "new-response" : "update-response", fullText);
isFirst = false; isFirst = false;
} }
} }
if (fullText.trim()) { if (fullText.trim()) {
localConversationHistory.push({ role: 'assistant', content: fullText.trim() }); localConversationHistory.push({
role: "assistant",
content: fullText.trim(),
});
saveConversationTurn(prompt, fullText); saveConversationTurn(prompt, fullText);
} }
console.log('[LocalAI] Image response completed'); console.log("[LocalAI] Image response completed");
sendToRenderer('update-status', 'Listening...'); sendToRenderer("update-status", "Listening...");
return { success: true, text: fullText, model: ollamaModel }; return { success: true, text: fullText, model: ollamaModel };
} catch (error) { } catch (error) {
console.error('[LocalAI] Image error:', error); console.error("[LocalAI] Image error:", error);
sendToRenderer('update-status', 'Ollama error: ' + error.message); sendToRenderer("update-status", "Ollama error: " + error.message);
return { success: false, error: error.message }; return { success: false, error: error.message };
} }
} }

150
src/utils/whisperWorker.js Normal file
View File

@ -0,0 +1,150 @@
/**
* Whisper Worker — runs ONNX Runtime in an isolated child process.
*
* The main Electron process forks this file and communicates via IPC messages.
* If ONNX Runtime crashes (SIGSEGV/SIGABRT inside the native Metal or CPU
* execution provider), only this worker dies — the main process survives and
* can respawn the worker automatically.
*
* Protocol (parent ↔ worker):
* parent → worker:
* { type: 'load', modelName, cacheDir }
* { type: 'transcribe', audioBase64 } // PCM 16-bit 16kHz as base64
* { type: 'shutdown' }
*
* worker → parent:
* { type: 'load-result', success, error? }
* { type: 'transcribe-result', success, text?, error? }
* { type: 'status', message }
* { type: 'ready' }
*/
let whisperPipeline = null;
function pcm16ToFloat32(pcm16Buffer) {
if (!pcm16Buffer || pcm16Buffer.length === 0) {
return new Float32Array(0);
}
const alignedLength =
pcm16Buffer.length % 2 === 0 ? pcm16Buffer.length : pcm16Buffer.length - 1;
const samples = alignedLength / 2;
const float32 = new Float32Array(samples);
for (let i = 0; i < samples; i++) {
float32[i] = pcm16Buffer.readInt16LE(i * 2) / 32768;
}
return float32;
}
async function loadModel(modelName, cacheDir) {
if (whisperPipeline) {
send({ type: "load-result", success: true });
return;
}
try {
send({
type: "status",
message: "Loading Whisper model (first time may take a while)...",
});
const { pipeline, env } = await import("@huggingface/transformers");
env.cacheDir = cacheDir;
whisperPipeline = await pipeline(
"automatic-speech-recognition",
modelName,
{
dtype: "q8",
device: "cpu",
},
);
send({ type: "load-result", success: true });
} catch (error) {
send({ type: "load-result", success: false, error: error.message });
}
}
async function transcribe(audioBase64) {
if (!whisperPipeline) {
send({
type: "transcribe-result",
success: false,
error: "Whisper pipeline not loaded",
});
return;
}
try {
const pcm16Buffer = Buffer.from(audioBase64, "base64");
if (pcm16Buffer.length < 2) {
send({
type: "transcribe-result",
success: false,
error: "Audio buffer too small",
});
return;
}
// Cap at ~30 seconds (16kHz, 16-bit mono)
const maxBytes = 16000 * 2 * 30;
const audioData =
pcm16Buffer.length > maxBytes
? pcm16Buffer.slice(0, maxBytes)
: pcm16Buffer;
const float32Audio = pcm16ToFloat32(audioData);
if (float32Audio.length === 0) {
send({
type: "transcribe-result",
success: false,
error: "Empty audio after conversion",
});
return;
}
const result = await whisperPipeline(float32Audio, {
sampling_rate: 16000,
language: "en",
task: "transcribe",
});
const text = result.text?.trim() || "";
send({ type: "transcribe-result", success: true, text });
} catch (error) {
send({
type: "transcribe-result",
success: false,
error: error.message || String(error),
});
}
}
function send(msg) {
try {
if (process.send) {
process.send(msg);
}
} catch (_) {
// Parent may have disconnected
}
}
process.on("message", (msg) => {
switch (msg.type) {
case "load":
loadModel(msg.modelName, msg.cacheDir).catch((err) => {
send({ type: "load-result", success: false, error: err.message });
});
break;
case "transcribe":
transcribe(msg.audioBase64).catch((err) => {
send({ type: "transcribe-result", success: false, error: err.message });
});
break;
case "shutdown":
process.exit(0);
break;
}
});
// Signal readiness to parent
send({ type: "ready" });