#!/usr/bin/env python3 """ Auto Clicker with AWS Rekognition Uses Amazon Rekognition for text detection and coordinate extraction. """ import os import sys import time import json import logging from typing import Tuple, Optional import base64 from io import BytesIO import pyautogui import boto3 from PIL import Image import argparse # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler('autoclicker_rekognition.log'), logging.StreamHandler(sys.stdout) ] ) logger = logging.getLogger(__name__) class AutoClickerRekognition: """Auto clicker using AWS Rekognition for text detection.""" def __init__(self, aws_access_key: str = None, aws_secret_key: str = None, region: str = 'us-east-1'): """ Initialize the Auto Clicker with AWS credentials. Args: aws_access_key (str): AWS access key aws_secret_key (str): AWS secret key region (str): AWS region """ try: # Initialize Rekognition client if aws_access_key and aws_secret_key: self.client = boto3.client( 'rekognition', aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key, region_name=region ) else: # Use default credentials (from environment or IAM role) self.client = boto3.client('rekognition', region_name=region) # Disable pyautogui failsafe for smoother operation pyautogui.FAILSAFE = True pyautogui.PAUSE = 0.1 logger.info("AWS Rekognition client initialized successfully") except Exception as e: logger.error(f"Error initializing AWS Rekognition client: {e}") raise def take_screenshot(self, region: Optional[Tuple[int, int, int, int]] = None) -> Image.Image: """ Take a screenshot of the screen or specified region. Args: region (Optional[Tuple[int, int, int, int]]): Region to capture (x, y, width, height) Returns: Image.Image: Screenshot as PIL Image """ try: if region: screenshot = pyautogui.screenshot(region=region) logger.info(f"Screenshot taken of region: {region}") else: screenshot = pyautogui.screenshot() logger.info("Full screen screenshot taken") return screenshot except Exception as e: logger.error(f"Error taking screenshot: {e}") raise def image_to_bytes(self, image: Image.Image) -> bytes: """ Convert PIL Image to bytes for AWS Rekognition. Args: image (Image.Image): PIL Image to convert Returns: bytes: Image bytes """ try: buffer = BytesIO() image.save(buffer, format='PNG') image_bytes = buffer.getvalue() logger.info("Image converted to bytes") return image_bytes except Exception as e: logger.error(f"Error converting image to bytes: {e}") raise def detect_text_with_rekognition(self, image: Image.Image, target_text: str) -> Optional[Tuple[int, int]]: """ Use AWS Rekognition to detect text and find coordinates. Args: image (Image.Image): Screenshot to analyze target_text (str): Text to find Returns: Optional[Tuple[int, int]]: Coordinates (x, y) or None if not found """ try: # Save screenshot for debugging debug_filename = f"debug_screenshot_{target_text}.png" image.save(debug_filename) logger.info(f"Debug screenshot saved as {debug_filename}") # Convert image to bytes image_bytes = self.image_to_bytes(image) # Call AWS Rekognition response = self.client.detect_text(Image={'Bytes': image_bytes}) logger.info(f"Rekognition found {len(response.get('TextDetections', []))} text detections") # Look for the target text target_text_lower = target_text.lower() best_match = None best_confidence = 0 logger.info(f"Looking for text: '{target_text}'") logger.info("All detected text:") for detection in response.get('TextDetections', []): detected_text = detection.get('DetectedText', '') confidence = detection.get('Confidence', 0) logger.info(f" - '{detected_text}' (confidence: {confidence})") # Check for exact match first, then partial match if detected_text.lower() == target_text_lower: logger.info(f"✅ Exact match found: '{detected_text}' (confidence: {confidence})") best_match = detection best_confidence = confidence break elif target_text_lower in detected_text.lower() and confidence > best_confidence: logger.info(f"⚠️ Partial match: '{detected_text}' (confidence: {confidence})") best_confidence = confidence best_match = detection if best_match: # Get bounding box coordinates bbox = best_match.get('Geometry', {}).get('BoundingBox', {}) if bbox: # Calculate center coordinates image_width, image_height = image.size left = int(bbox.get('Left', 0) * image_width) top = int(bbox.get('Top', 0) * image_height) width = int(bbox.get('Width', 0) * image_width) height = int(bbox.get('Height', 0) * image_height) # Center coordinates center_x = left + (width // 2) center_y = top + (height // 2) logger.info(f"Found '{target_text}' at ({center_x}, {center_y}) with confidence: {best_confidence}") logger.info(f"Bounding box: left={left}, top={top}, width={width}, height={height}") return (center_x, center_y) else: logger.warning("No bounding box found in detection") return None else: logger.warning(f"Text '{target_text}' not found in image") return None except Exception as e: logger.error(f"Error detecting text with Rekognition: {e}") return None def click_at_coordinates(self, x: int, y: int, button: str = 'left') -> bool: """ Click at the specified coordinates. Args: x (int): X coordinate y (int): Y coordinate button (str): Mouse button to click ('left', 'right', 'middle') Returns: bool: True if click was successful """ try: # Move mouse to coordinates first pyautogui.moveTo(x, y, duration=0.5) time.sleep(0.2) # Click at the coordinates pyautogui.click(x, y, button=button) logger.info(f"Clicked at coordinates ({x}, {y}) with {button} button") return True except Exception as e: logger.error(f"Error clicking at coordinates ({x}, {y}): {e}") return False def process_screenshot_and_click(self, text_prompt: str, region: Optional[Tuple[int, int, int, int]] = None) -> bool: """ Complete workflow: take screenshot, analyze with Rekognition, and click. Args: text_prompt (str): Text description of what to find and click region (Optional[Tuple[int, int, int, int]]): Region to capture Returns: bool: True if successful """ try: logger.info(f"Starting process for text: '{text_prompt}'") # Take screenshot screenshot = self.take_screenshot(region) # Get coordinates from Rekognition coordinates = self.detect_text_with_rekognition(screenshot, text_prompt) if coordinates: x, y = coordinates # Click at the coordinates success = self.click_at_coordinates(x, y) if success: logger.info("Process completed successfully") return True else: logger.error("Failed to click at coordinates") return False else: logger.error("Could not find element in screenshot") return False except Exception as e: logger.error(f"Error in process_screenshot_and_click: {e}") return False def load_config() -> dict: """Load configuration from config.json file.""" config_path = 'config.json' if os.path.exists(config_path): try: with open(config_path, 'r') as f: return json.load(f) except Exception as e: logger.error(f"Error loading config: {e}") return {} def main(): """Main function to run the auto clicker.""" parser = argparse.ArgumentParser(description='Auto Clicker with AWS Rekognition') parser.add_argument('--text', '-t', required=True, help='Text description of what to find and click') parser.add_argument('--region', '-r', nargs=4, type=int, metavar=('X', 'Y', 'WIDTH', 'HEIGHT'), help='Region to capture (x y width height)') parser.add_argument('--aws-access-key', help='AWS Access Key') parser.add_argument('--aws-secret-key', help='AWS Secret Key') parser.add_argument('--aws-region', default='us-east-1', help='AWS Region') parser.add_argument('--delay', '-d', type=float, default=2.0, help='Delay before taking screenshot (seconds)') args = parser.parse_args() # Load configuration config = load_config() # Get AWS credentials (optional if using local IAM) aws_access_key = (args.aws_access_key or config.get('aws_access_key') or os.getenv('AWS_ACCESS_KEY_ID')) aws_secret_key = (args.aws_secret_key or config.get('aws_secret_key') or os.getenv('AWS_SECRET_ACCESS_KEY')) # Initialize auto clicker (will use local IAM if no explicit credentials) try: if aws_access_key and aws_secret_key: clicker = AutoClickerRekognition(aws_access_key, aws_secret_key, args.aws_region) logger.info("Using explicit AWS credentials") else: clicker = AutoClickerRekognition(region=args.aws_region) logger.info("Using local IAM configuration") except Exception as e: logger.error(f"Failed to initialize AWS Rekognition: {e}") logger.error("Please check your AWS configuration:") logger.error("1. Run 'aws configure' to set up credentials") logger.error("2. Or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables") logger.error("3. Or add credentials to config.json") sys.exit(1) # Add delay if specified if args.delay > 0: logger.info(f"Waiting {args.delay} seconds before taking screenshot...") time.sleep(args.delay) # Process the screenshot and click region = tuple(args.region) if args.region else None success = clicker.process_screenshot_and_click(args.text, region) if success: logger.info("Auto clicker completed successfully") sys.exit(0) else: logger.error("Auto clicker failed") sys.exit(1) if __name__ == "__main__": main()