from typing import List, Optional, Tuple from .constants import ( USE_GRID_CLICKING, GRID_COLS, GRID_ROWS, GRID_PROMPT_MODE, GRID_LABEL_FONT_SIZE, GRID_COL_LABEL_POSITIONS, GRID_USE_HEADER_BANDS, GRID_RENDERER, GRID_ALPHA_CELL_SIZE, ) from .utils import column_index_to_label def build_find_value_prompt(value_text: str, screen_width: int, screen_height: int, row_hint: Optional[str] = None) -> Tuple[str, List[str]]: if USE_GRID_CLICKING and GRID_RENDERER == 'alpha_numeric': # Authoritative, coordinate-based grid rules (no OCR of labels) cell_px = int(GRID_ALPHA_CELL_SIZE) cols = max(1, int(screen_width // cell_px)) rows = max(1, int(screen_height // cell_px)) prompt = ( "You are an assistant that returns ONE grid location for a target object in a screenshot.\n" f"Target: {value_text}.\n\n" "Grid indexing and ID rules (authoritative):\n" f"- Cell size: {cell_px}x{cell_px} px. Image width is {int(screen_width)} px ⇒ grid_cols = {cols} (rows = {rows} for height {int(screen_height)} px).\n" "- Draw a tight bounding box around ONLY the exact target substring (exclude symbols, commas, decimals, units, or surrounding text).\n" "- Let (x,y) be the center of this box (pixels from top-left of the image).\n" f"- Compute zero-based indices:\n - col = floor(x / {cell_px})\n - row = floor(y / {cell_px})\n" f"- Compute the grid ID:\n - grid_id = zero-pad-3(row * {cols} + col)\n" "- Do NOT read or trust the faint grid label text; always compute the ID from coordinates.\n" "- Derive cell_position from the box center within its cell using thirds: {top, center, bottom} × {left, center, right}.\n" "- Validation:\n - verified = true only if the box tightly encloses exactly the target substring and the chosen cell matches the computed grid_id.\n\n" "Output (strict JSON only):\n" '{"grid_id": "<000-999>", "cell_position": "", "verified": true, "grid_row": 0, "grid_column": 0, "box": [x1,y1,x2,y2], "coordinates": [x,y], "matched_text": "", "reason": ""}' ) elif USE_GRID_CLICKING: # Geometry-only mapping (no header OCR). We trust the substring box to derive the grid cell. end_label = column_index_to_label(GRID_COLS) or "Z" # Compute the exact pixel geometry of the drawn grid so the model can map rows/cols deterministically header_top = int(GRID_LABEL_FONT_SIZE * 1.6) if GRID_USE_HEADER_BANDS else 0 footer_bottom = int(GRID_LABEL_FONT_SIZE * 1.6) if (GRID_USE_HEADER_BANDS and GRID_COL_LABEL_POSITIONS in ("bottom", "both")) else 0 left_gutter = int(GRID_LABEL_FONT_SIZE * 1.8) if GRID_USE_HEADER_BANDS else 0 grid_w = max(1, int(screen_width) - left_gutter) grid_h = max(1, int(screen_height) - header_top - footer_bottom) cell_w = grid_w / GRID_COLS cell_h = grid_h / GRID_ROWS prompt = ( "You are an assistant that returns ONE grid location for a target string in a screenshot.\n" f"Target: {value_text}.\n" "Interpretation:\n" "- Treat minor formatting differences as equivalent (case, spaces, thin spaces, separators, currency symbols), but the character order must match exactly.\n" "- If the target appears inside a longer string, isolate the exact substring characters of the target only (e.g., for '440' inside 'A$440.00', select only '440').\n" "Grid:\n" f"- Overlay a {GRID_COLS}x{GRID_ROWS} grid on the full screenshot (columns 1..{GRID_COLS} and A..{end_label}, rows 1..{GRID_ROWS}). Columns are labeled NUMERIC at the TOP and BOTTOM; rows are labeled NUMERIC on the LEFT.\n" "- Draw a tight bounding box around the target substring only.\n" "- Choose the grid cell whose CENTER lies inside that box.\n" "- If the box overlaps two rows, choose the TOP row. If multiple candidates exist, choose the first by position: top-most, then left-most.\n" "- Derive cell_position from the tight-box center within the chosen cell using thirds (top/center/bottom and left/center/right) and combine into one of the 9 positions.\n" "- Determine grid_column by READING the numeric column header at the BOTTOM, directly below the box center (do not estimate by proportion).\n" "- Determine grid_row by READING the numeric row header on the LEFT aligned with the TOP edge (y1) of the box.\n" "Validation checklist (double-check before answering):\n" "- Verify the characters inside the box exactly match the target substring (exclude currency symbols, commas, extra digits, or surrounding text).\n" "- If the box contents do not match exactly, adjust the box to the correct substring and update the grid cell accordingly.\n" "- If no exact match exists, return the best single cell and explain the discrepancy in 'reason'.\n" "Final confirmation: set 'verified' to true only if the box tightly encloses exactly the target substring and the chosen cell is correct; otherwise false and explain why.\n" "Output (strict JSON only, no prose, no code fences). Include a brief reason. Include cell_position as one of: top-left, top-center, top-right, center-left, center, center-right, bottom-left, bottom-center, bottom-right.\n" '{"grid_id": "AA27", "cell_position": "center", "verified": true, "grid_row": 27, "grid_column": 27, "box": [x1,y1,x2,y2], "coordinates": [x,y], "matched_text": "...", "reason": "why this substring and which headers (BOTTOM/LEFT) or geometry were used"}' ) else: # Fallback non-grid path (unchanged) prompt = ( "You are an assistant that identifies UI text on a desktop screenshot.\n" f"Find the text that matches the target value: {value_text}.\n" "Treat common formatting variations as equivalent (case, whitespace, separators, symbols); the character order must match.\n" "OUTPUT (JSON only): coordinates ([x,y] or null), box ([x1,y1,x2,y2] or null), matched_text (string or null), reason (string).\n" f"Coordinates must be within [0,{screen_width}]x[0,{screen_height}]." ) aux_texts: List[str] = [] if row_hint: aux_texts.append(f"HINT={row_hint}") aux_texts.append(f"TARGET_VALUE={value_text}") return prompt, aux_texts # --- Links analysis prompt ---------------------------------------------------- def get_links_prompt(screen_width: int, screen_height: int) -> str: """Return the standardized prompt for supporting-document links analysis. Args: screen_width: Current screen width in pixels screen_height: Current screen height in pixels Returns: A string prompt instructing the model to return only a JSON array of supporting-document links. """ sw = int(screen_width) sh = int(screen_height) return ( """ Analyze this screenshot and identify ONLY invoice/payment/bill/receipt PAGE TEXT LINKS (navigation links), not file downloads. It is possible there are ZERO qualifying links on this screen. Look for ONLY navigation TEXT LINKS that lead to invoice/payment/receipt/bill pages. Acceptable examples (illustrative, not exhaustive): - "View invoice" - "View receipt" - "Open payment details" - "Statement" - "Credit note" - Reference number links in tables (e.g., blue linked numbers or phrases like "(No reference number)") Do NOT return static labels or plain values. Non-link examples (must be excluded): - "Invoice number" label text - "INV-12345" or "Invoice #12345" shown as plain text - "Payment 25 Jan 2023" shown as plain text (black/gray, not underlined) Return ONLY a JSON array with this exact format (the array may be EMPTY if no links exist): [ {{ "type": "supporting_document_link", "description": "brief description of what the link provides", "link_words": "the exact visible text of the link (verbatim)", "coordinates": [x, y], "confidence": 0.95, "button": false }} ] Rules: - link_words MUST be the exact words in the clickable link, verbatim, including punctuation (e.g., Payment (25 Jan 2023)) - If multiple words compose the link, include the full visible phrase as link_words - RETURN ALL qualifying links visible on the screen. Create ONE ARRAY ITEM PER LINK. - The number of returned items MUST equal the count of qualifying links; do a careful top-to-bottom scan and do not skip any. - If there are multiple qualifying links in the same row/column (e.g., reference numbers for different entries), include EACH as a separate item. - EXCLUDE buttons, badges, tabs, or icon-only elements (e.g., "Files (3)", paperclip icons, PDF icons) - button: MUST be false. Do NOT return buttons or icon-only elements - x and y must be pixel coordinates (0-{{xmax}} for x, 0-{{ymax}} for y) - ONLY include text links that navigate to invoice/payment/receipt/bill pages - The returned item MUST be a clickable text link (not static text). Prefer underlined or link-colored text; if unsure, EXCLUDE it - EXCLUDE black or near-black/plain text (e.g., typical body text). Prefer link-like colors (blue or brand-accent). If the text looks black/gray, EXCLUDE it - EXCLUDE links that are not document-related (e.g., "Get set up now", "Settings", "Help", marketing or onboarding CTAs) - EXCLUDE actions for receiving or recording payments (e.g., "Receive Payment", "Record payment", "Receive a payment") - EXCLUDE controls to add or configure payment options/methods (e.g., "Add payment options", "Add payment method(s)") - EXCLUDE file downloads and filenames (.pdf, .png, .jpg, .jpeg, .tif, .tiff, .doc/.docx, .xls/.xlsx) - EXCLUDE links with the exact words "View details" (any case) - EXCLUDE links containing the phrase "Activity history" (any case) - EXCLUDE entity/contact/company name links that appear under or are associated with a label/column "From" (e.g., supplier/customer/company names) - If NO qualifying links are present, return an EMPTY array [] (do NOT invent any links) - Be precise with coordinates - Return valid JSON only - Focus on invoice-related supporting materials - Do not include controls for new documents such as "Attach files", "Upload", or add/plus icons - Do not include print actions (e.g., "Print PDF", "Print") - This screenshot is {{sw}}x{{sh}} pixels """ ).format(xmax=sw-1, ymax=sh-1, sw=sw, sh=sh) def get_attachment_links_prompt(screen_width: int, screen_height: int) -> str: """Generic attachments-focused supporting-docs prompt for non-Xero apps.""" sw = int(screen_width) sh = int(screen_height) return ( """ Analyze this screenshot and identify ONLY attachment-related links to download files. Look for: - Download links or buttons for attachments - "View" or "Download" buttons that open files - File attachment icons (paperclip), PDF/document icons - Links that clearly indicate a downloadable file (receipt, invoice, statement, document) Return ONLY a JSON array with this exact format: [ { "type": "supporting_document_link", "description": "brief description", "link_words": "the exact visible text (verbatim) or empty if icon-only", "coordinates": [x, y], "confidence": 0.95, "button": true } ] Rules: - Exclude actions that upload or attach new files (e.g., "Attach files", "Upload") - Exclude print actions (e.g., "Print", "Print PDF") - link_words must be exact when present; set to "" if icon-only - x and y must be pixel coordinates (0-{xmax} for x, 0-{ymax} for y) - Return valid JSON only - This screenshot is {sw}x{sh} pixels """ ).format(xmax=sw-1, ymax=sh-1, sw=sw, sh=sh) def get_attachment_presence_and_names_prompt(screen_width: int, screen_height: int) -> str: """Prompt to detect if an attachments dialog has downloadable items and list their clickable names. Returns JSON ONLY in the following schema: { "modal_present": true, "has_attachments": true, "links": ["sage.pdf", "another_file.png"] } Notes: - links should be the exact clickable text users would click to open the file (e.g., the file name like 'sage.pdf'). - Exclude non-file controls: 'Add Attachment', 'Delete All', 'Delete', 'Close', and any explanatory text. - If icons exist with no text, omit them (this prompt focuses on text names for Rekognition to click). - modal_present is false if no attachments dialog is visible. - has_attachments is true only if there is at least one downloadable item in the list. """ sw = int(screen_width) sh = int(screen_height) return ( """ You are analyzing a desktop screenshot that may show an "Attachments" dialog. Task: 1) Determine if an attachments dialog is visible (look for a title like "Attachments" and a list/grid of files). 2) If visible, determine whether there are any downloadable attachments listed. 3) If there are, extract the exact clickable text of each attachment (typically the file names like 'sage.pdf'). STRICT OUTPUT (JSON only; no prose, no code blocks): { "modal_present": true|false, "has_attachments": true|false, "links": [""] } Rules: - links must be an array of strings, each being the exact visible text a user would click to open the file (e.g., 'sage.pdf'). - Exclude controls or boilerplate text such as 'Add Attachment', 'Delete All', 'Delete', 'Close', size limits and usage notes. - Do NOT include print actions. - If no dialog is visible, set modal_present=false and has_attachments=false and links=[]. - If the dialog is visible but shows zero attachments, set modal_present=true, has_attachments=false, links=[]. - Coordinates are NOT required; only names. - The screenshot size is {sw}x{sh} pixels. """ ).format(sw=sw, sh=sh) def get_sd_icons_prompt(screen_width: int, screen_height: int) -> str: """Return the specialized prompt for SD (Supporting Document) icon detection. Args: screen_width: Current screen width in pixels screen_height: Current screen height in pixels Returns: A string prompt instructing the model to return only a JSON array of SD icons. """ sw = int(screen_width) sh = int(screen_height) return ( """ Analyze this screenshot and identify ONLY SD (Supporting Document) icons that might be found on invoices. Look specifically for: - Document icons (rectangular with folded corner, like the example) - PDF icons (document icon with PDF text or symbol) - File attachment icons (paperclip, document with attachment symbol) - Download icons (arrow pointing down, download symbol) - Receipt icons (document with receipt-like appearance) - Invoice attachment icons (document with invoice symbol) - Any clickable document-related icons Focus on icons that are: - Rectangular document shapes with folded top-right corner - Small, clickable icon elements (not large text blocks) - Clearly document or file-related in appearance - Likely to contain supporting documents when clicked Return ONLY a JSON array with this exact format: [ {{ "type": "sd_icon", "description": "brief description of the icon (e.g., 'PDF document icon', 'Receipt icon')", "link_words": "any visible text on or near the icon (empty string if icon-only)", "coordinates": [x, y], "confidence": 0.95, "button": true }} ] Rules: - link_words: If the icon has visible text, include it exactly. If icon-only, set to "" (empty string) - button: Always set to true for SD icons (they are clickable elements) - x and y must be pixel coordinates (0-{{xmax}} for x, 0-{{ymax}} for y) - ONLY include actual document/file icons, not text links or buttons - Be precise with coordinates (center of the icon) - Return valid JSON only - Focus on small, clickable icon elements - Do not include large text blocks or non-icon elements - This screenshot is {{sw}}x{{sh}} pixels """ ).format(xmax=sw-1, ymax=sh-1, sw=sw, sh=sh)