diff --git a/src/components/app/AppHeader.js b/src/components/app/AppHeader.js index f0211bd..c58bf7d 100644 --- a/src/components/app/AppHeader.js +++ b/src/components/app/AppHeader.js @@ -120,6 +120,72 @@ export class AppHeader extends LitElement { .update-button:hover { background: rgba(241, 76, 76, 0.1); } + + .status-wrapper { + position: relative; + display: inline-flex; + align-items: center; + } + + .status-text { + font-size: var(--header-font-size-small); + color: var(--text-secondary); + max-width: 120px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .status-text.error { + color: #f14c4c; + } + + .status-tooltip { + position: absolute; + top: 100%; + right: 0; + margin-top: 8px; + background: var(--tooltip-bg, #1a1a1a); + color: var(--tooltip-text, #ffffff); + padding: 10px 14px; + border-radius: 6px; + font-size: 12px; + max-width: 300px; + word-wrap: break-word; + white-space: normal; + opacity: 0; + visibility: hidden; + transition: opacity 0.15s ease, visibility 0.15s ease; + pointer-events: none; + box-shadow: 0 4px 12px rgba(0,0,0,0.3); + z-index: 1000; + line-height: 1.4; + } + + .status-tooltip::before { + content: ''; + position: absolute; + bottom: 100%; + right: 16px; + border: 6px solid transparent; + border-bottom-color: var(--tooltip-bg, #1a1a1a); + } + + .status-wrapper:hover .status-tooltip { + opacity: 1; + visibility: visible; + } + + .status-tooltip .tooltip-label { + font-size: 10px; + text-transform: uppercase; + opacity: 0.6; + margin-bottom: 4px; + } + + .status-tooltip .tooltip-content { + color: #f14c4c; + } `; static properties = { @@ -273,6 +339,8 @@ export class AppHeader extends LitElement { render() { const elapsedTime = this.getElapsedTime(); + const isError = this.statusText && (this.statusText.toLowerCase().includes('error') || this.statusText.toLowerCase().includes('failed')); + const shortStatus = isError ? 'Error' : this.statusText; return html`
@@ -281,7 +349,15 @@ export class AppHeader extends LitElement { ${this.currentView === 'assistant' ? html` ${elapsedTime} - ${this.statusText} +
+ ${shortStatus} + ${isError ? html` +
+
Error Details
+
${this.statusText}
+
+ ` : ''} +
${this.isClickThrough ? html`click-through` : ''} ` : ''} diff --git a/src/components/views/AssistantView.js b/src/components/views/AssistantView.js index fa1e35a..1628263 100644 --- a/src/components/views/AssistantView.js +++ b/src/components/views/AssistantView.js @@ -314,6 +314,40 @@ export class AssistantView extends LitElement { opacity: 0.5; font-size: 10px; } + + .capture-buttons { + display: flex; + gap: 6px; + } + + .region-select-btn { + display: flex; + align-items: center; + justify-content: center; + background: transparent; + color: var(--text-secondary); + border: 1px solid var(--border-color); + padding: 6px 10px; + border-radius: 20px; + font-size: 12px; + cursor: pointer; + transition: all 0.15s ease; + } + + .region-select-btn:hover { + background: var(--hover-background); + color: var(--text-color); + border-color: var(--text-color); + } + + .region-select-btn svg { + width: 16px; + height: 16px; + } + + .region-select-btn span { + margin-left: 4px; + } `; static properties = { @@ -546,6 +580,14 @@ export class AssistantView extends LitElement { } } + handleRegionSelect() { + if (window.startRegionSelection) { + window.startRegionSelection(); + // Reload limits after a short delay to catch the update + setTimeout(() => this.loadLimits(), 1000); + } + } + scrollToBottom() { setTimeout(() => { const container = this.shadowRoot.querySelector('.response-container'); @@ -608,25 +650,33 @@ export class AssistantView extends LitElement { -
-
-
- Flash - ${this.flashCount}/20 -
-
- Flash Lite - ${this.flashLiteCount}/20 -
-
Resets every 24 hours
-
- +
+
+
+ Flash + ${this.flashCount}/20 +
+
+ Flash Lite + ${this.flashLiteCount}/20 +
+
Resets every 24 hours
+
+ +
`; diff --git a/src/utils/gemini.js b/src/utils/gemini.js index 4f6186c..36ce507 100644 --- a/src/utils/gemini.js +++ b/src/utils/gemini.js @@ -265,8 +265,9 @@ async function initializeGeminiSession(apiKey, customPrompt = '', profile = 'int } }, onerror: function (e) { - console.log('Session error:', e.message); - sendToRenderer('update-status', 'Error: ' + e.message); + const errorMsg = e?.message || e?.error?.message || 'Session error occurred'; + console.log('Session error:', errorMsg, e); + sendToRenderer('update-status', 'Error: ' + errorMsg); }, onclose: function (e) { console.log('Session closed:', e.reason); diff --git a/src/utils/prompts.js b/src/utils/prompts.js index b7160b4..caf8d76 100644 --- a/src/utils/prompts.js +++ b/src/utils/prompts.js @@ -219,7 +219,60 @@ function getSystemPrompt(profile, customPrompt = '', googleSearchEnabled = true) return buildSystemPrompt(promptParts, customPrompt, googleSearchEnabled); } +// Comprehensive prompt for Vision/Image analysis +const VISION_ANALYSIS_PROMPT = `You are an expert AI assistant analyzing a screenshot. Your task is to understand what the user needs help with and provide the most useful response. + +**ANALYSIS APPROACH:** +1. First, identify what's shown on the screen (code editor, math problem, website, document, exam, etc.) +2. Determine what the user likely needs (explanation, solution, answer, debugging help, etc.) +3. Provide a direct, actionable response + +**RESPONSE GUIDELINES BY CONTEXT:** + +**If it's CODE (LeetCode, HackerRank, coding interview, IDE):** +- Identify the programming language and problem type +- Provide a brief explanation of the approach (2-3 bullet points max) +- Give the complete, working code solution +- Include time/space complexity if relevant +- If there's an error, explain the fix + +**If it's MATH or SCIENCE:** +- Show step-by-step solution +- Use proper mathematical notation with LaTeX ($..$ for inline, $$...$$ for blocks) +- Provide the final answer clearly marked +- Include any relevant formulas used + +**If it's MCQ/EXAM/QUIZ:** +- State the correct answer immediately and clearly (e.g., "**Answer: B**") +- Provide brief justification (1-2 sentences) +- If multiple questions visible, answer all of them + +**If it's a DOCUMENT/ARTICLE/WEBSITE:** +- Summarize the key information +- Answer any specific questions if apparent +- Highlight important points + +**If it's a FORM/APPLICATION:** +- Help fill in the required information +- Suggest appropriate responses +- Point out any issues or missing fields + +**If it's an ERROR/DEBUG scenario:** +- Identify the error type and cause +- Provide the fix immediately +- Explain briefly why it occurred + +**FORMAT REQUIREMENTS:** +- Use **markdown** for formatting +- Use **bold** for key answers and important points +- Use code blocks with language specification for code +- Be concise but complete - no unnecessary explanations +- No pleasantries or filler text - get straight to the answer + +**CRITICAL:** Provide the complete answer. Don't ask for clarification - make reasonable assumptions and deliver value immediately.`; + module.exports = { profilePrompts, getSystemPrompt, + VISION_ANALYSIS_PROMPT, }; diff --git a/src/utils/renderer.js b/src/utils/renderer.js index e1620cb..10d1edb 100644 --- a/src/utils/renderer.js +++ b/src/utils/renderer.js @@ -153,7 +153,7 @@ async function initializeGemini(profile = 'interview', language = 'en-US') { if (success) { cheatingDaddy.setStatus('Live'); } else { - cheatingDaddy.setStatus('error'); + cheatingDaddy.setStatus('Error: Failed to initialize AI session Gemini'); } } @@ -329,7 +329,21 @@ async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'mediu console.log('Manual mode enabled - screenshots will be captured on demand only'); } catch (err) { console.error('Error starting capture:', err); - cheatingDaddy.setStatus('error'); + + // Provide more helpful error messages based on error type + let errorMessage = err.message || 'Failed to start capture'; + + if (errorMessage.toLowerCase().includes('timeout')) { + errorMessage = 'Screen capture timed out. Please try again and select a screen quickly.'; + } else if (errorMessage.toLowerCase().includes('permission') || errorMessage.toLowerCase().includes('denied')) { + errorMessage = 'Screen capture permission denied. Please grant screen recording permission in System Settings.'; + } else if (errorMessage.toLowerCase().includes('not found') || errorMessage.toLowerCase().includes('no sources')) { + errorMessage = 'No screen sources found. Please ensure a display is connected.'; + } else if (errorMessage.toLowerCase().includes('aborted') || errorMessage.toLowerCase().includes('cancel')) { + errorMessage = 'Screen selection was cancelled. Please try again.'; + } + + cheatingDaddy.setStatus('Error: ' + errorMessage); } } @@ -518,10 +532,193 @@ async function captureScreenshot(imageQuality = 'medium', isManual = false) { ); } -const MANUAL_SCREENSHOT_PROMPT = `Help me on this page, give me the answer no bs, complete answer. -So if its a code question, give me the approach in few bullet points, then the entire code. Also if theres anything else i need to know, tell me. -If its a question about the website, give me the answer no bs, complete answer. -If its a mcq question, give me the answer no bs, complete answer.`; +const MANUAL_SCREENSHOT_PROMPT = `You are an expert AI assistant analyzing a screenshot. Your task is to understand what the user needs help with and provide the most useful response. + +**ANALYSIS APPROACH:** +1. First, identify what's shown on the screen (code editor, math problem, website, document, exam, etc.) +2. Determine what the user likely needs (explanation, solution, answer, debugging help, etc.) +3. Provide a direct, actionable response + +**RESPONSE GUIDELINES BY CONTEXT:** + +**If it's CODE (LeetCode, HackerRank, coding interview, IDE):** +- Identify the programming language and problem type +- Provide a brief explanation of the approach (2-3 bullet points max) +- Give the complete, working code solution +- Include time/space complexity if relevant +- If there's an error, explain the fix + +**If it's MATH or SCIENCE:** +- Show step-by-step solution +- Use proper mathematical notation with LaTeX ($..$ for inline, $$...$$ for blocks) +- Provide the final answer clearly marked +- Include any relevant formulas used + +**If it's MCQ/EXAM/QUIZ:** +- State the correct answer immediately and clearly (e.g., "**Answer: B**") +- Provide brief justification (1-2 sentences) +- If multiple questions visible, answer all of them + +**If it's a DOCUMENT/ARTICLE/WEBSITE:** +- Summarize the key information +- Answer any specific questions if apparent +- Highlight important points + +**If it's a FORM/APPLICATION:** +- Help fill in the required information +- Suggest appropriate responses +- Point out any issues or missing fields + +**If it's an ERROR/DEBUG scenario:** +- Identify the error type and cause +- Provide the fix immediately +- Explain briefly why it occurred + +**FORMAT REQUIREMENTS:** +- Use **markdown** for formatting +- Use **bold** for key answers and important points +- Use code blocks with language specification for code +- Be concise but complete - no unnecessary explanations +- No pleasantries or filler text - get straight to the answer + +**CRITICAL:** Provide the complete answer. Don't ask for clarification - make reasonable assumptions and deliver value immediately.`; + +// ============ REGION SELECTION ============ +// Uses a separate fullscreen window to allow selection outside the app window + +async function startRegionSelection() { + console.log('Starting region selection...'); + + if (!mediaStream) { + console.error('No media stream available. Please start capture first.'); + cheatingDaddy?.addNewResponse('Please start screen capture first before selecting a region.'); + return; + } + + // Ensure video is ready + if (!hiddenVideo) { + hiddenVideo = document.createElement('video'); + hiddenVideo.srcObject = mediaStream; + hiddenVideo.muted = true; + hiddenVideo.playsInline = true; + await hiddenVideo.play(); + + await new Promise(resolve => { + if (hiddenVideo.readyState >= 2) return resolve(); + hiddenVideo.onloadedmetadata = () => resolve(); + }); + + // Initialize canvas + offscreenCanvas = document.createElement('canvas'); + offscreenCanvas.width = hiddenVideo.videoWidth; + offscreenCanvas.height = hiddenVideo.videoHeight; + offscreenContext = offscreenCanvas.getContext('2d'); + } + + if (hiddenVideo.readyState < 2) { + console.warn('Video not ready yet'); + return; + } + + // Capture current screen to show in selection window + offscreenContext.drawImage(hiddenVideo, 0, 0, offscreenCanvas.width, offscreenCanvas.height); + const screenshotDataUrl = offscreenCanvas.toDataURL('image/jpeg', 0.9); + + // Request main process to create selection window + const result = await ipcRenderer.invoke('start-region-selection', { screenshotDataUrl }); + + if (result.success && result.rect) { + // Capture the selected region from the screenshot + await captureRegionFromScreenshot(result.rect, screenshotDataUrl); + } else if (result.cancelled) { + console.log('Region selection cancelled'); + } else if (result.error) { + console.error('Region selection error:', result.error); + } +} + +async function captureRegionFromScreenshot(rect, screenshotDataUrl) { + console.log('Capturing region from screenshot:', rect); + + // Load the screenshot into an image + const img = new Image(); + img.src = screenshotDataUrl; + + await new Promise((resolve, reject) => { + img.onload = resolve; + img.onerror = reject; + }); + + // Calculate scale factor (screenshot might have different resolution than display) + // The selection coordinates are in screen pixels, we need to map to image pixels + const scaleX = img.naturalWidth / window.screen.width; + const scaleY = img.naturalHeight / window.screen.height; + + // Scale the selection rectangle + const scaledRect = { + left: Math.round(rect.left * scaleX), + top: Math.round(rect.top * scaleY), + width: Math.round(rect.width * scaleX), + height: Math.round(rect.height * scaleY), + }; + + // Create canvas for the cropped region + const cropCanvas = document.createElement('canvas'); + cropCanvas.width = scaledRect.width; + cropCanvas.height = scaledRect.height; + const cropContext = cropCanvas.getContext('2d'); + + // Draw only the selected region + cropContext.drawImage( + img, + scaledRect.left, + scaledRect.top, + scaledRect.width, + scaledRect.height, + 0, + 0, + scaledRect.width, + scaledRect.height + ); + + // Convert to blob and send + cropCanvas.toBlob( + async blob => { + if (!blob) { + console.error('Failed to create blob from cropped region'); + return; + } + + const reader = new FileReader(); + reader.onloadend = async () => { + const base64data = reader.result.split(',')[1]; + + if (!base64data || base64data.length < 100) { + console.error('Invalid base64 data generated'); + return; + } + + const result = await ipcRenderer.invoke('send-image-content', { + data: base64data, + prompt: MANUAL_SCREENSHOT_PROMPT, + }); + + if (result.success) { + console.log(`Region capture response completed from ${result.model}`); + } else { + console.error('Failed to get region capture response:', result.error); + cheatingDaddy.addNewResponse(`Error: ${result.error}`); + } + }; + reader.readAsDataURL(blob); + }, + 'image/jpeg', + 0.9 + ); +} + +// Expose to global scope +window.startRegionSelection = startRegionSelection; async function captureManualScreenshot(imageQuality = null) { console.log('Manual screenshot triggered'); diff --git a/src/utils/window.js b/src/utils/window.js index efe0e37..9cff896 100644 --- a/src/utils/window.js +++ b/src/utils/window.js @@ -33,14 +33,59 @@ function createWindow(sendToRenderer, geminiSessionRef) { }); const { session, desktopCapturer } = require('electron'); - session.defaultSession.setDisplayMediaRequestHandler( - (request, callback) => { - desktopCapturer.getSources({ types: ['screen'] }).then(sources => { - callback({ video: sources[0], audio: 'loopback' }); - }); - }, - { useSystemPicker: true } - ); + + // Setup display media request handler for screen capture + // On macOS, use system picker for better UX + if (process.platform === 'darwin') { + session.defaultSession.setDisplayMediaRequestHandler( + async (request, callback) => { + try { + const sources = await desktopCapturer.getSources({ + types: ['screen'], + thumbnailSize: { width: 0, height: 0 } // Skip thumbnail generation for speed + }); + + if (sources.length === 0) { + console.error('No screen sources available'); + callback(null); + return; + } + + // On macOS, directly use the first screen (system already granted permission) + console.log('Screen capture source:', sources[0].name); + callback({ video: sources[0], audio: 'loopback' }); + } catch (error) { + console.error('Error getting screen sources:', error); + callback(null); + } + }, + { useSystemPicker: false } // Disable system picker, use our source directly + ); + } else { + // On other platforms, use the system picker + session.defaultSession.setDisplayMediaRequestHandler( + async (request, callback) => { + try { + const sources = await desktopCapturer.getSources({ + types: ['screen'], + thumbnailSize: { width: 0, height: 0 } + }); + + if (sources.length === 0) { + console.error('No screen sources available'); + callback(null); + return; + } + + callback({ video: sources[0], audio: 'loopback' }); + } catch (error) { + console.error('Error getting screen sources:', error); + callback(null); + } + }, + { useSystemPicker: true } + ); + } mainWindow.setResizable(false); mainWindow.setContentProtection(true); @@ -493,6 +538,221 @@ function setupWindowIpcHandlers(mainWindow, sendToRenderer, geminiSessionRef) { return { success: false, error: error.message }; } }); + + // Region selection window for capturing areas outside the main window + let regionSelectionWindow = null; + + ipcMain.handle('start-region-selection', async (event, { screenshotDataUrl }) => { + try { + // Hide main window first + const wasVisible = mainWindow.isVisible(); + if (wasVisible) { + mainWindow.hide(); + } + + // Small delay to ensure window is hidden + await new Promise(resolve => setTimeout(resolve, 100)); + + // Get all displays to cover all screens + const displays = screen.getAllDisplays(); + const primaryDisplay = screen.getPrimaryDisplay(); + + // Calculate bounds that cover all displays + let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; + displays.forEach(display => { + minX = Math.min(minX, display.bounds.x); + minY = Math.min(minY, display.bounds.y); + maxX = Math.max(maxX, display.bounds.x + display.bounds.width); + maxY = Math.max(maxY, display.bounds.y + display.bounds.height); + }); + + const totalWidth = maxX - minX; + const totalHeight = maxY - minY; + + // Create fullscreen transparent window for selection + regionSelectionWindow = new BrowserWindow({ + x: minX, + y: minY, + width: totalWidth, + height: totalHeight, + frame: false, + transparent: true, + alwaysOnTop: true, + skipTaskbar: true, + resizable: false, + movable: false, + hasShadow: false, + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + }, + }); + + regionSelectionWindow.setAlwaysOnTop(true, 'screen-saver', 1); + + // Create HTML content for selection overlay + const htmlContent = ` + + + + + + + +
+
+
Click and drag to select region • ESC to cancel
+ + + + `; + + regionSelectionWindow.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(htmlContent)}`); + + return new Promise((resolve) => { + ipcMain.once('region-selected', (event, rect) => { + if (regionSelectionWindow && !regionSelectionWindow.isDestroyed()) { + regionSelectionWindow.close(); + regionSelectionWindow = null; + } + if (wasVisible) { + mainWindow.showInactive(); + } + resolve({ success: true, rect }); + }); + + ipcMain.once('region-selection-cancelled', () => { + if (regionSelectionWindow && !regionSelectionWindow.isDestroyed()) { + regionSelectionWindow.close(); + regionSelectionWindow = null; + } + if (wasVisible) { + mainWindow.showInactive(); + } + resolve({ success: false, cancelled: true }); + }); + + // Also handle window close + regionSelectionWindow.on('closed', () => { + regionSelectionWindow = null; + if (wasVisible && !mainWindow.isDestroyed()) { + mainWindow.showInactive(); + } + }); + }); + } catch (error) { + console.error('Error starting region selection:', error); + if (regionSelectionWindow && !regionSelectionWindow.isDestroyed()) { + regionSelectionWindow.close(); + regionSelectionWindow = null; + } + if (!mainWindow.isDestroyed()) { + mainWindow.showInactive(); + } + return { success: false, error: error.message }; + } + }); } module.exports = {