Add custom screen picker dialog for Windows audio capture and update version to 0.5.10

package.json

@@ -1,7 +1,7 @@
 {
     "name": "cheating-daddy",
     "productName": "cheating-daddy",
-    "version": "0.5.9",
+    "version": "0.5.10",
     "description": "cheating daddy",
     "main": "src/index.js",
     "scripts": {

src/components/app/CheatingDaddyApp.js

@@ -6,6 +6,7 @@ import { HelpView } from '../views/HelpView.js';
 import { HistoryView } from '../views/HistoryView.js';
 import { AssistantView } from '../views/AssistantView.js';
 import { OnboardingView } from '../views/OnboardingView.js';
+import { ScreenPickerDialog } from '../views/ScreenPickerDialog.js';
 
 export class CheatingDaddyApp extends LitElement {
     static styles = css`
@@ -112,6 +113,8 @@ export class CheatingDaddyApp extends LitElement {
         _storageLoaded: { state: true },
         aiProvider: { type: String },
         modelInfo: { type: Object },
+        showScreenPicker: { type: Boolean },
+        screenSources: { type: Array },
     };
 
     constructor() {
@@ -137,6 +140,8 @@ export class CheatingDaddyApp extends LitElement {
         this._storageLoaded = false;
         this.aiProvider = 'gemini';
         this.modelInfo = { model: '', visionModel: '', whisperModel: '' };
+        this.showScreenPicker = false;
+        this.screenSources = [];
 
         // Load from storage
         this._loadFromStorage();
@@ -549,6 +554,16 @@ export class CheatingDaddyApp extends LitElement {
                         <div class="view-container">${this.renderCurrentView()}</div>
                     </div>
                 </div>
+                ${this.showScreenPicker
+                    ? html`
+                          <screen-picker-dialog
+                              ?visible=${this.showScreenPicker}
+                              .sources=${this.screenSources}
+                              @source-selected=${this.handleSourceSelected}
+                              @cancelled=${this.handlePickerCancelled}
+                          ></screen-picker-dialog>
+                      `
+                    : ''}
             </div>
         `;
     }
@@ -579,6 +594,44 @@ export class CheatingDaddyApp extends LitElement {
 
         this.requestUpdate();
     }
+
+    async showScreenPickerDialog() {
+        const { ipcRenderer } = window.require('electron');
+        const result = await ipcRenderer.invoke('get-screen-sources');
+        
+        if (result.success) {
+            this.screenSources = result.sources;
+            this.showScreenPicker = true;
+            return new Promise((resolve) => {
+                this._screenPickerResolve = resolve;
+            });
+        } else {
+            console.error('Failed to get screen sources:', result.error);
+            return { cancelled: true };
+        }
+    }
+
+    async handleSourceSelected(event) {
+        const { source } = event.detail;
+        const { ipcRenderer } = window.require('electron');
+        
+        // Tell main process which source was selected
+        await ipcRenderer.invoke('set-selected-source', source.id);
+        
+        this.showScreenPicker = false;
+        if (this._screenPickerResolve) {
+            this._screenPickerResolve({ source });
+            this._screenPickerResolve = null;
+        }
+    }
+
+    handlePickerCancelled() {
+        this.showScreenPicker = false;
+        if (this._screenPickerResolve) {
+            this._screenPickerResolve({ cancelled: true });
+            this._screenPickerResolve = null;
+        }
+    }
 }
 
 customElements.define('cheating-daddy-app', CheatingDaddyApp);

src/components/views/ScreenPickerDialog.js

@@ -0,0 +1,175 @@
+import { html, css, LitElement } from '../../assets/lit-core-2.7.4.min.js';
+
+export class ScreenPickerDialog extends LitElement {
+    static properties = {
+        sources: { type: Array },
+        visible: { type: Boolean },
+    };
+
+    static styles = css`
+        :host {
+            display: none;
+            position: fixed;
+            top: 0;
+            left: 0;
+            right: 0;
+            bottom: 0;
+            background: rgba(0, 0, 0, 0.8);
+            z-index: 10000;
+            align-items: center;
+            justify-content: center;
+        }
+
+        :host([visible]) {
+            display: flex;
+        }
+
+        .dialog {
+            background: var(--background-color);
+            border: 1px solid var(--border-color);
+            border-radius: 8px;
+            padding: 24px;
+            max-width: 800px;
+            max-height: 80vh;
+            overflow-y: auto;
+        }
+
+        h2 {
+            margin: 0 0 16px 0;
+            color: var(--text-color);
+            font-size: 18px;
+            font-weight: 500;
+        }
+
+        .sources-grid {
+            display: grid;
+            grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
+            gap: 12px;
+            margin-bottom: 16px;
+        }
+
+        .source-item {
+            background: var(--input-background);
+            border: 2px solid transparent;
+            border-radius: 6px;
+            padding: 12px;
+            cursor: pointer;
+            transition: all 0.2s ease;
+        }
+
+        .source-item:hover {
+            border-color: var(--border-default);
+            background: var(--button-hover);
+        }
+
+        .source-item.selected {
+            border-color: var(--accent-color);
+            background: var(--button-hover);
+        }
+
+        .source-thumbnail {
+            width: 100%;
+            height: 120px;
+            object-fit: contain;
+            background: #1a1a1a;
+            border-radius: 4px;
+            margin-bottom: 8px;
+        }
+
+        .source-name {
+            color: var(--text-color);
+            font-size: 13px;
+            text-align: center;
+            word-break: break-word;
+        }
+
+        .buttons {
+            display: flex;
+            gap: 8px;
+            justify-content: flex-end;
+        }
+
+        button {
+            background: var(--button-background);
+            color: var(--text-color);
+            border: 1px solid var(--border-color);
+            padding: 8px 16px;
+            border-radius: 3px;
+            cursor: pointer;
+            font-size: 13px;
+            transition: background-color 0.1s ease;
+        }
+
+        button:hover {
+            background: var(--button-hover);
+        }
+
+        button.primary {
+            background: var(--accent-color);
+            color: white;
+            border-color: var(--accent-color);
+        }
+
+        button.primary:hover {
+            background: var(--accent-hover);
+        }
+
+        button:disabled {
+            opacity: 0.5;
+            cursor: not-allowed;
+        }
+    `;
+
+    constructor() {
+        super();
+        this.sources = [];
+        this.visible = false;
+        this.selectedSource = null;
+    }
+
+    selectSource(source) {
+        this.selectedSource = source;
+        this.requestUpdate();
+    }
+
+    confirm() {
+        if (this.selectedSource) {
+            this.dispatchEvent(
+                new CustomEvent('source-selected', {
+                    detail: { source: this.selectedSource },
+                })
+            );
+        }
+    }
+
+    cancel() {
+        this.dispatchEvent(new CustomEvent('cancelled'));
+    }
+
+    render() {
+        return html`
+            <div class="dialog">
+                <h2>Choose screen or window to share</h2>
+                <div class="sources-grid">
+                    ${this.sources.map(
+                        source => html`
+                            <div
+                                class="source-item ${this.selectedSource?.id === source.id ? 'selected' : ''}"
+                                @click=${() => this.selectSource(source)}
+                            >
+                                <img class="source-thumbnail" src="${source.thumbnail}" alt="${source.name}" />
+                                <div class="source-name">${source.name}</div>
+                            </div>
+                        `
+                    )}
+                </div>
+                <div class="buttons">
+                    <button @click=${this.cancel}>Cancel</button>
+                    <button class="primary" @click=${this.confirm} ?disabled=${!this.selectedSource}>Share</button>
+                </div>
+            </div>
+        `;
+    }
+}
+
+customElements.define('screen-picker-dialog', ScreenPickerDialog);

src/utils/renderer.js

@@ -294,7 +294,21 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
 
             console.log('Linux capture started - system audio:', mediaStream.getAudioTracks().length > 0, 'microphone mode:', audioMode);
         } else {
-            // Windows - use display media with loopback for system audio
+            // Windows - show custom screen picker first
+            logToMain('info', '=== Starting Windows audio capture ===');
+            cheatingDaddy.setStatus('Choose screen to share...');
+            
+            // Show screen picker dialog
+            const appElement = document.querySelector('cheating-daddy-app');
+            const pickerResult = await appElement.showScreenPickerDialog();
+            
+            if (pickerResult.cancelled) {
+                cheatingDaddy.setStatus('Cancelled');
+                return;
+            }
+            
+            cheatingDaddy.setStatus('Starting capture...');
+            
             mediaStream = await navigator.mediaDevices.getDisplayMedia({
                 video: {
                     frameRate: 1,
@@ -310,7 +324,6 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
                 },
             });
 
-            console.log('Windows capture started with loopback audio');
             const audioTracks = mediaStream.getAudioTracks();
             const videoTracks = mediaStream.getVideoTracks();
             
@@ -326,8 +339,14 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu
                 })),
             });
 
-            // Setup audio processing for Windows loopback audio only
-            setupWindowsLoopbackProcessing();
+            if (audioTracks.length === 0) {
+                logToMain('warn', 'WARNING: No audio tracks! User must check "Share audio" in screen picker dialog');
+                cheatingDaddy.setStatus('Warning: No audio - enable "Share audio" checkbox');
+            } else {
+                logToMain('info', 'Audio track acquired, setting up processing...');
+                // Setup audio processing for Windows loopback audio only
+                setupWindowsLoopbackProcessing();
+            }
 
             if (audioMode === 'mic_only' || audioMode === 'both') {
                 let micStream = null;
@@ -443,32 +462,73 @@ function setupLinuxSystemAudioProcessing() {
 
 function setupWindowsLoopbackProcessing() {
     // Setup audio processing for Windows loopback audio only
-    audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
-    const source = audioContext.createMediaStreamSource(mediaStream);
-    audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
+    logToMain('info', 'Setting up Windows loopback audio processing...');
     
-    let audioBuffer = [];
-    const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
+    try {
+        audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
         
-    audioProcessor.onaudioprocess = async e => {
-        const inputData = e.inputBuffer.getChannelData(0);
-        audioBuffer.push(...inputData);
+        logToMain('info', 'AudioContext created:', {
+            state: audioContext.state,
+            sampleRate: audioContext.sampleRate,
+        });
         
-        // Process audio in chunks
-        while (audioBuffer.length >= samplesPerChunk) {
-            const chunk = audioBuffer.splice(0, samplesPerChunk);
-            const pcmData16 = convertFloat32ToInt16(chunk);
-            const base64Data = arrayBufferToBase64(pcmData16.buffer);
-
-            await ipcRenderer.invoke('send-audio-content', {
-                data: base64Data,
-                mimeType: 'audio/pcm;rate=24000',
+        // Resume AudioContext if suspended (Chrome policy)
+        if (audioContext.state === 'suspended') {
+            logToMain('warn', 'AudioContext suspended, attempting resume...');
+            audioContext.resume().then(() => {
+                logToMain('info', 'AudioContext resumed successfully');
+            }).catch(err => {
+                logToMain('error', 'Failed to resume AudioContext:', err.message);
             });
         }
-    };
         
-    source.connect(audioProcessor);
-    audioProcessor.connect(audioContext.destination);
+        const source = audioContext.createMediaStreamSource(mediaStream);
+        audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
+
+        let audioBuffer = [];
+        const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
+        let chunkCount = 0;
+        let totalSamples = 0;
+
+        audioProcessor.onaudioprocess = async e => {
+            const inputData = e.inputBuffer.getChannelData(0);
+            audioBuffer.push(...inputData);
+            totalSamples += inputData.length;
+
+            // Process audio in chunks
+            while (audioBuffer.length >= samplesPerChunk) {
+                const chunk = audioBuffer.splice(0, samplesPerChunk);
+                const pcmData16 = convertFloat32ToInt16(chunk);
+                const base64Data = arrayBufferToBase64(pcmData16.buffer);
+
+                await ipcRenderer.invoke('send-audio-content', {
+                    data: base64Data,
+                    mimeType: 'audio/pcm;rate=24000',
+                });
+                
+                chunkCount++;
+                
+                // Log progress every 100 chunks (~10 seconds)
+                if (chunkCount === 1) {
+                    logToMain('info', 'First audio chunk sent to AI');
+                    cheatingDaddy.setStatus('Listening...');
+                } else if (chunkCount % 100 === 0) {
+                    // Calculate max amplitude to check if we're getting real audio
+                    const maxAmp = Math.max(...chunk.map(Math.abs));
+                    logToMain('info', `Audio progress: ${chunkCount} chunks, maxAmplitude: ${maxAmp.toFixed(4)}`);
+                }
+            }
+        };
+
+        source.connect(audioProcessor);
+        audioProcessor.connect(audioContext.destination);
+        
+        logToMain('info', 'Windows audio processing pipeline connected');
+        
+    } catch (err) {
+        logToMain('error', 'Error setting up Windows audio:', err.message, err.stack);
+        cheatingDaddy.setStatus('Audio error: ' + err.message);
+    }
 }
 
 async function captureScreenshot(imageQuality = 'medium', isManual = false) {

src/utils/window.js

@@ -33,14 +33,54 @@ function createWindow(sendToRenderer, geminiSessionRef) {
     });
 
     const { session, desktopCapturer } = require('electron');
-    session.defaultSession.setDisplayMediaRequestHandler(
-        (request, callback) => {
-            desktopCapturer.getSources({ types: ['screen'] }).then(sources => {
-                callback({ video: sources[0], audio: 'loopback' });
-            });
-        },
-        { useSystemPicker: true }
-    );
+    
+    // Store selected source for Windows custom picker
+    let selectedSourceId = null;
+    
+    // Setup display media handler based on platform
+    if (process.platform === 'darwin') {
+        // macOS: Use native system picker
+        session.defaultSession.setDisplayMediaRequestHandler(
+            (request, callback) => {
+                desktopCapturer.getSources({ types: ['screen'] }).then(sources => {
+                    callback({ video: sources[0], audio: 'loopback' });
+                });
+            },
+            { useSystemPicker: true }
+        );
+    } else {
+        // Windows/Linux: Use selected source from custom picker
+        session.defaultSession.setDisplayMediaRequestHandler(async (request, callback) => {
+            try {
+                const sources = await desktopCapturer.getSources({
+                    types: ['screen', 'window'],
+                    thumbnailSize: { width: 0, height: 0 },
+                });
+
+                // Find the selected source or use first screen
+                let source = sources[0];
+                if (selectedSourceId) {
+                    const found = sources.find(s => s.id === selectedSourceId);
+                    if (found) source = found;
+                }
+
+                if (source) {
+                    callback({ video: source, audio: 'loopback' });
+                } else {
+                    callback({});
+                }
+            } catch (error) {
+                console.error('Error in display media handler:', error);
+                callback({});
+            }
+        });
+    }
+
+    // IPC handler to set selected source
+    ipcMain.handle('set-selected-source', async (event, sourceId) => {
+        selectedSourceId = sourceId;
+        return { success: true };
+    });
 
     mainWindow.setResizable(false);
     mainWindow.setContentProtection(true);
@@ -715,6 +755,29 @@ function setupWindowIpcHandlers(mainWindow, sendToRenderer, geminiSessionRef) {
             return { success: false, error: error.message };
         }
     });
+
+    // Get available screen sources for picker
+    ipcMain.handle('get-screen-sources', async () => {
+        try {
+            const { desktopCapturer } = require('electron');
+            const sources = await desktopCapturer.getSources({
+                types: ['screen', 'window'],
+                thumbnailSize: { width: 150, height: 150 },
+            });
+
+            return {
+                success: true,
+                sources: sources.map(source => ({
+                    id: source.id,
+                    name: source.name,
+                    thumbnail: source.thumbnail.toDataURL(),
+                })),
+            };
+        } catch (error) {
+            console.error('Error getting screen sources:', error);
+            return { success: false, error: error.message };
+        }
+    });
 }
 
 module.exports = {