diff --git a/package.json b/package.json index 383f713..79cb90c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cheating-daddy", "productName": "cheating-daddy", - "version": "0.5.9", + "version": "0.5.10", "description": "cheating daddy", "main": "src/index.js", "scripts": { diff --git a/src/components/app/CheatingDaddyApp.js b/src/components/app/CheatingDaddyApp.js index 1fde960..313b3e5 100644 --- a/src/components/app/CheatingDaddyApp.js +++ b/src/components/app/CheatingDaddyApp.js @@ -6,6 +6,7 @@ import { HelpView } from '../views/HelpView.js'; import { HistoryView } from '../views/HistoryView.js'; import { AssistantView } from '../views/AssistantView.js'; import { OnboardingView } from '../views/OnboardingView.js'; +import { ScreenPickerDialog } from '../views/ScreenPickerDialog.js'; export class CheatingDaddyApp extends LitElement { static styles = css` @@ -112,6 +113,8 @@ export class CheatingDaddyApp extends LitElement { _storageLoaded: { state: true }, aiProvider: { type: String }, modelInfo: { type: Object }, + showScreenPicker: { type: Boolean }, + screenSources: { type: Array }, }; constructor() { @@ -137,6 +140,8 @@ export class CheatingDaddyApp extends LitElement { this._storageLoaded = false; this.aiProvider = 'gemini'; this.modelInfo = { model: '', visionModel: '', whisperModel: '' }; + this.showScreenPicker = false; + this.screenSources = []; // Load from storage this._loadFromStorage(); @@ -549,6 +554,16 @@ export class CheatingDaddyApp extends LitElement {
${this.renderCurrentView()}
+ ${this.showScreenPicker + ? html` + + ` + : ''} `; } @@ -579,6 +594,44 @@ export class CheatingDaddyApp extends LitElement { this.requestUpdate(); } + + async showScreenPickerDialog() { + const { ipcRenderer } = window.require('electron'); + const result = await ipcRenderer.invoke('get-screen-sources'); + + if (result.success) { + this.screenSources = result.sources; + this.showScreenPicker = true; + return new Promise((resolve) => { + this._screenPickerResolve = resolve; + }); + } else { + console.error('Failed to get screen sources:', result.error); + return { cancelled: true }; + } + } + + async handleSourceSelected(event) { + const { source } = event.detail; + const { ipcRenderer } = window.require('electron'); + + // Tell main process which source was selected + await ipcRenderer.invoke('set-selected-source', source.id); + + this.showScreenPicker = false; + if (this._screenPickerResolve) { + this._screenPickerResolve({ source }); + this._screenPickerResolve = null; + } + } + + handlePickerCancelled() { + this.showScreenPicker = false; + if (this._screenPickerResolve) { + this._screenPickerResolve({ cancelled: true }); + this._screenPickerResolve = null; + } + } } customElements.define('cheating-daddy-app', CheatingDaddyApp); diff --git a/src/components/views/ScreenPickerDialog.js b/src/components/views/ScreenPickerDialog.js new file mode 100644 index 0000000..8552d8e --- /dev/null +++ b/src/components/views/ScreenPickerDialog.js @@ -0,0 +1,175 @@ +import { html, css, LitElement } from '../../assets/lit-core-2.7.4.min.js'; + +export class ScreenPickerDialog extends LitElement { + static properties = { + sources: { type: Array }, + visible: { type: Boolean }, + }; + + static styles = css` + :host { + display: none; + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.8); + z-index: 10000; + align-items: center; + justify-content: center; + } + + :host([visible]) { + display: flex; + } + + .dialog { + background: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 8px; + padding: 24px; + max-width: 800px; + max-height: 80vh; + overflow-y: auto; + } + + h2 { + margin: 0 0 16px 0; + color: var(--text-color); + font-size: 18px; + font-weight: 500; + } + + .sources-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 12px; + margin-bottom: 16px; + } + + .source-item { + background: var(--input-background); + border: 2px solid transparent; + border-radius: 6px; + padding: 12px; + cursor: pointer; + transition: all 0.2s ease; + } + + .source-item:hover { + border-color: var(--border-default); + background: var(--button-hover); + } + + .source-item.selected { + border-color: var(--accent-color); + background: var(--button-hover); + } + + .source-thumbnail { + width: 100%; + height: 120px; + object-fit: contain; + background: #1a1a1a; + border-radius: 4px; + margin-bottom: 8px; + } + + .source-name { + color: var(--text-color); + font-size: 13px; + text-align: center; + word-break: break-word; + } + + .buttons { + display: flex; + gap: 8px; + justify-content: flex-end; + } + + button { + background: var(--button-background); + color: var(--text-color); + border: 1px solid var(--border-color); + padding: 8px 16px; + border-radius: 3px; + cursor: pointer; + font-size: 13px; + transition: background-color 0.1s ease; + } + + button:hover { + background: var(--button-hover); + } + + button.primary { + background: var(--accent-color); + color: white; + border-color: var(--accent-color); + } + + button.primary:hover { + background: var(--accent-hover); + } + + button:disabled { + opacity: 0.5; + cursor: not-allowed; + } + `; + + constructor() { + super(); + this.sources = []; + this.visible = false; + this.selectedSource = null; + } + + selectSource(source) { + this.selectedSource = source; + this.requestUpdate(); + } + + confirm() { + if (this.selectedSource) { + this.dispatchEvent( + new CustomEvent('source-selected', { + detail: { source: this.selectedSource }, + }) + ); + } + } + + cancel() { + this.dispatchEvent(new CustomEvent('cancelled')); + } + + render() { + return html` +
+

Choose screen or window to share

+
+ ${this.sources.map( + source => html` +
this.selectSource(source)} + > + ${source.name} +
${source.name}
+
+ ` + )} +
+
+ + +
+
+ `; + } +} + +customElements.define('screen-picker-dialog', ScreenPickerDialog); diff --git a/src/utils/renderer.js b/src/utils/renderer.js index 27f8e1d..e60c4be 100644 --- a/src/utils/renderer.js +++ b/src/utils/renderer.js @@ -294,7 +294,21 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu console.log('Linux capture started - system audio:', mediaStream.getAudioTracks().length > 0, 'microphone mode:', audioMode); } else { - // Windows - use display media with loopback for system audio + // Windows - show custom screen picker first + logToMain('info', '=== Starting Windows audio capture ==='); + cheatingDaddy.setStatus('Choose screen to share...'); + + // Show screen picker dialog + const appElement = document.querySelector('cheating-daddy-app'); + const pickerResult = await appElement.showScreenPickerDialog(); + + if (pickerResult.cancelled) { + cheatingDaddy.setStatus('Cancelled'); + return; + } + + cheatingDaddy.setStatus('Starting capture...'); + mediaStream = await navigator.mediaDevices.getDisplayMedia({ video: { frameRate: 1, @@ -310,7 +324,6 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu }, }); - console.log('Windows capture started with loopback audio'); const audioTracks = mediaStream.getAudioTracks(); const videoTracks = mediaStream.getVideoTracks(); @@ -326,8 +339,14 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu })), }); - // Setup audio processing for Windows loopback audio only - setupWindowsLoopbackProcessing(); + if (audioTracks.length === 0) { + logToMain('warn', 'WARNING: No audio tracks! User must check "Share audio" in screen picker dialog'); + cheatingDaddy.setStatus('Warning: No audio - enable "Share audio" checkbox'); + } else { + logToMain('info', 'Audio track acquired, setting up processing...'); + // Setup audio processing for Windows loopback audio only + setupWindowsLoopbackProcessing(); + } if (audioMode === 'mic_only' || audioMode === 'both') { let micStream = null; @@ -443,32 +462,73 @@ function setupLinuxSystemAudioProcessing() { function setupWindowsLoopbackProcessing() { // Setup audio processing for Windows loopback audio only - audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); - const source = audioContext.createMediaStreamSource(mediaStream); - audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); - - let audioBuffer = []; - const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; - - audioProcessor.onaudioprocess = async e => { - const inputData = e.inputBuffer.getChannelData(0); - audioBuffer.push(...inputData); - - // Process audio in chunks - while (audioBuffer.length >= samplesPerChunk) { - const chunk = audioBuffer.splice(0, samplesPerChunk); - const pcmData16 = convertFloat32ToInt16(chunk); - const base64Data = arrayBufferToBase64(pcmData16.buffer); - - await ipcRenderer.invoke('send-audio-content', { - data: base64Data, - mimeType: 'audio/pcm;rate=24000', + logToMain('info', 'Setting up Windows loopback audio processing...'); + + try { + audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); + + logToMain('info', 'AudioContext created:', { + state: audioContext.state, + sampleRate: audioContext.sampleRate, + }); + + // Resume AudioContext if suspended (Chrome policy) + if (audioContext.state === 'suspended') { + logToMain('warn', 'AudioContext suspended, attempting resume...'); + audioContext.resume().then(() => { + logToMain('info', 'AudioContext resumed successfully'); + }).catch(err => { + logToMain('error', 'Failed to resume AudioContext:', err.message); }); } - }; + + const source = audioContext.createMediaStreamSource(mediaStream); + audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1); - source.connect(audioProcessor); - audioProcessor.connect(audioContext.destination); + let audioBuffer = []; + const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION; + let chunkCount = 0; + let totalSamples = 0; + + audioProcessor.onaudioprocess = async e => { + const inputData = e.inputBuffer.getChannelData(0); + audioBuffer.push(...inputData); + totalSamples += inputData.length; + + // Process audio in chunks + while (audioBuffer.length >= samplesPerChunk) { + const chunk = audioBuffer.splice(0, samplesPerChunk); + const pcmData16 = convertFloat32ToInt16(chunk); + const base64Data = arrayBufferToBase64(pcmData16.buffer); + + await ipcRenderer.invoke('send-audio-content', { + data: base64Data, + mimeType: 'audio/pcm;rate=24000', + }); + + chunkCount++; + + // Log progress every 100 chunks (~10 seconds) + if (chunkCount === 1) { + logToMain('info', 'First audio chunk sent to AI'); + cheatingDaddy.setStatus('Listening...'); + } else if (chunkCount % 100 === 0) { + // Calculate max amplitude to check if we're getting real audio + const maxAmp = Math.max(...chunk.map(Math.abs)); + logToMain('info', `Audio progress: ${chunkCount} chunks, maxAmplitude: ${maxAmp.toFixed(4)}`); + } + } + }; + + source.connect(audioProcessor); + audioProcessor.connect(audioContext.destination); + + logToMain('info', 'Windows audio processing pipeline connected'); + + } catch (err) { + logToMain('error', 'Error setting up Windows audio:', err.message, err.stack); + cheatingDaddy.setStatus('Audio error: ' + err.message); + } } async function captureScreenshot(imageQuality = 'medium', isManual = false) { diff --git a/src/utils/window.js b/src/utils/window.js index c461cb7..691c7ef 100644 --- a/src/utils/window.js +++ b/src/utils/window.js @@ -33,14 +33,54 @@ function createWindow(sendToRenderer, geminiSessionRef) { }); const { session, desktopCapturer } = require('electron'); - session.defaultSession.setDisplayMediaRequestHandler( - (request, callback) => { - desktopCapturer.getSources({ types: ['screen'] }).then(sources => { - callback({ video: sources[0], audio: 'loopback' }); - }); - }, - { useSystemPicker: true } - ); + + // Store selected source for Windows custom picker + let selectedSourceId = null; + + // Setup display media handler based on platform + if (process.platform === 'darwin') { + // macOS: Use native system picker + session.defaultSession.setDisplayMediaRequestHandler( + (request, callback) => { + desktopCapturer.getSources({ types: ['screen'] }).then(sources => { + callback({ video: sources[0], audio: 'loopback' }); + }); + }, + { useSystemPicker: true } + ); + } else { + // Windows/Linux: Use selected source from custom picker + session.defaultSession.setDisplayMediaRequestHandler(async (request, callback) => { + try { + const sources = await desktopCapturer.getSources({ + types: ['screen', 'window'], + thumbnailSize: { width: 0, height: 0 }, + }); + + // Find the selected source or use first screen + let source = sources[0]; + if (selectedSourceId) { + const found = sources.find(s => s.id === selectedSourceId); + if (found) source = found; + } + + if (source) { + callback({ video: source, audio: 'loopback' }); + } else { + callback({}); + } + } catch (error) { + console.error('Error in display media handler:', error); + callback({}); + } + }); + } + + // IPC handler to set selected source + ipcMain.handle('set-selected-source', async (event, sourceId) => { + selectedSourceId = sourceId; + return { success: true }; + }); mainWindow.setResizable(false); mainWindow.setContentProtection(true); @@ -715,6 +755,29 @@ function setupWindowIpcHandlers(mainWindow, sendToRenderer, geminiSessionRef) { return { success: false, error: error.message }; } }); + + // Get available screen sources for picker + ipcMain.handle('get-screen-sources', async () => { + try { + const { desktopCapturer } = require('electron'); + const sources = await desktopCapturer.getSources({ + types: ['screen', 'window'], + thumbnailSize: { width: 150, height: 150 }, + }); + + return { + success: true, + sources: sources.map(source => ({ + id: source.id, + name: source.name, + thumbnail: source.thumbnail.toDataURL(), + })), + }; + } catch (error) { + console.error('Error getting screen sources:', error); + return { success: false, error: error.message }; + } + }); } module.exports = {