当目标区域内有参照物的图片字样出现。就触发鼠标对应的动作,可以是单击,双击,右击。
功能场景:
1、可以用来刷网课,比如网课看了10分钟会弹出一个确认或继续学习的按钮。
2、各种软件及电商倒计时出现点击字样,可以全自动处理。
3、可二次开发更多功能...
Python代码:
- import tkinter as tk
- from tkinter import messagebox, ttk
- import ttkbootstrap as ttkb
- from ttkbootstrap.constants import *
- import cv2
- import numpy as np
- import pyautogui
- import threading
- import time
- import os
- from PIL import Image, ImageTk
- import keyboard
- import win32gui
- import win32con
-
-
- class ScreenCaptureApp:
- def __init__(self, root):
- self.root = root
- self.root.title("opencv--图片比对触发动作V1.0 开发:许小小仙")
- self.root.geometry("700x500")
- self.root.minsize(700, 500)
-
- # 设置中文字体支持
- self.style = ttkb.Style(theme='flatly')
- self.style.configure('.', font=('SimHei', 10))
-
- # 初始化变量
- self.reference_image = None
- self.target_image = None
- self.reference_region = None # 参照物区域坐标
- self.target_region = None # 目标区域坐标
- self.comparison_thread = None
- self.is_comparing = False
- self.stop_event = threading.Event()
-
- # 记录软件窗口信息
- self.window_handle = None
- self.window_rect = None
-
- # 创建界面
- self.create_widgets()
-
- # 注册热键
- keyboard.add_hotkey('f8', self.toggle_comparison)
-
- # 获取窗口句柄
- self.update_window_handle()
-
- def update_window_handle(self):
- """更新窗口句柄和位置信息"""
-
- def callback(hwnd, extra):
- if win32gui.IsWindowVisible(hwnd):
- if "图像对比与鼠标自动操作工具" in win32gui.GetWindowText(hwnd):
- extra.append(hwnd)
- return True
-
- windows = []
- win32gui.EnumWindows(callback, windows)
-
- if windows:
- self.window_handle = windows[0]
- self.update_window_rect()
-
- def update_window_rect(self):
- """更新窗口位置信息"""
- if self.window_handle:
- rect = win32gui.GetWindowRect(self.window_handle)
- self.window_rect = rect
-
- def create_widgets(self):
- # 创建主框架
- main_frame = ttkb.Frame(self.root, padding=10)
- main_frame.pack(fill=BOTH, expand=True)
-
- # 顶部操作区
- top_frame = ttkb.Frame(main_frame, padding=10)
- top_frame.pack(fill=X, pady=5)
-
- # 参照物截图按钮
- ttkb.Button(
- top_frame,
- text="截取参照物区域",
- command=self.capture_reference,
- bootstyle=SUCCESS,
- width=15
- ).pack(side=LEFT, padx=5)
-
- # 目标区域截图按钮
- ttkb.Button(
- top_frame,
- text="截取目标区域",
- command=self.capture_target,
- bootstyle=INFO,
- width=15
- ).pack(side=LEFT, padx=5)
-
- # 开始/停止对比按钮
- self.compare_button = ttkb.Button(
- top_frame,
- text="开始对比 (F8)",
- command=self.toggle_comparison,
- bootstyle=PRIMARY,
- width=15
- )
- self.compare_button.pack(side=LEFT, padx=5)
-
- # 清空按钮
- ttkb.Button(
- top_frame,
- text="清空所有",
- command=self.clear_all,
- bootstyle=DANGER,
- width=10
- ).pack(side=LEFT, padx=5)
-
- # 图像显示区域
- images_frame = ttkb.Frame(main_frame, padding=10)
- images_frame.pack(fill=BOTH, expand=True, pady=5)
-
- # 左侧 - 参照物图像 (固定大小300x200)
- reference_frame = ttkb.LabelFrame(images_frame, text="参照物区域", padding=10)
- reference_frame.pack(side=LEFT, fill=BOTH, expand=True, padx=(0, 5))
-
- # 创建固定大小的画布
- self.reference_canvas = tk.Canvas(reference_frame, width=300, height=200, bg="white")
- self.reference_canvas.pack(fill=BOTH, expand=True)
-
- self.reference_label = ttk.Label(self.reference_canvas, text="未选择参照物区域")
- self.reference_label.place(relx=0.5, rely=0.5, anchor="center")
-
- # 右侧 - 目标图像 (固定大小300x200)
- target_frame = ttkb.LabelFrame(images_frame, text="目标区域 (实时更新)", padding=10)
- target_frame.pack(side=RIGHT, fill=BOTH, expand=True, padx=(5, 0))
-
- # 创建固定大小的画布
- self.target_canvas = tk.Canvas(target_frame, width=300, height=200, bg="white")
- self.target_canvas.pack(fill=BOTH, expand=True)
-
- self.target_label = ttk.Label(self.target_canvas, text="未选择目标区域")
- self.target_label.place(relx=0.5, rely=0.5, anchor="center")
-
- # 底部控制区域 - 拆分为两行
- # 第一行:鼠标动作和触发方式
- control_row1 = ttkb.Frame(main_frame, padding=10)
- control_row1.pack(fill=X, pady=(5, 0))
-
- # 鼠标动作选择
- ttkb.Label(control_row1, text="触发动作:").pack(side=LEFT, padx=5)
- self.mouse_action = tk.StringVar(value="单击")
- ttkb.Combobox(
- control_row1,
- textvariable=self.mouse_action,
- values=["单击", "双击", "右击"],
- state="readonly",
- width=8
- ).pack(side=LEFT, padx=5)
-
- # 触发方式选择
- ttkb.Label(control_row1, text="触发方式:").pack(side=LEFT, padx=5)
- self.trigger_mode = tk.StringVar(value="触发一次")
- ttkb.Combobox(
- control_row1,
- textvariable=self.trigger_mode,
- values=["触发一次", "无限触发", "触发一次过10秒后再次检测"],
- state="readonly",
- width=20
- ).pack(side=LEFT, padx=5)
-
- # 第二行:相似度阈值
- control_row2 = ttkb.Frame(main_frame, padding=10)
- control_row2.pack(fill=X, pady=(0, 5))
-
- # 相似度阈值
- ttkb.Label(control_row2, text="相似度阈值:").pack(side=LEFT, padx=5)
- self.threshold = tk.DoubleVar(value=0.8)
- ttkb.Scale(
- control_row2,
- variable=self.threshold,
- from_=0.5,
- to=1.0,
- orient=HORIZONTAL,
- length=150,
- bootstyle=PRIMARY
- ).pack(side=LEFT, padx=5)
-
- self.threshold_label = ttkb.Label(control_row2, text=f"{self.threshold.get():.2f}")
- self.threshold_label.pack(side=LEFT, padx=5)
-
- self.threshold.trace_add("write", lambda *args: self.threshold_label.config(text=f"{self.threshold.get():.2f}"))
-
- # 状态条
- self.status_var = tk.StringVar(value="就绪")
- self.status_bar = ttkb.Label(main_frame, textvariable=self.status_var, relief=SUNKEN, anchor=W)
- self.status_bar.pack(side=BOTTOM, fill=X)
-
- # 窗口位置变化时更新窗口信息
- self.root.bind("<Configure>", lambda e: self.update_window_rect())
-
- def capture_reference(self):
- """截取参照物区域"""
- self.status_var.set("请在屏幕上框选参照物区域...")
- self.root.iconify() # 最小化主窗口
- time.sleep(0.5) # 等待窗口最小化完成
- region = self._capture_screen_region()
- self.root.deiconify() # 恢复主窗口
- self.root.lift() # 提升窗口层级
-
- if region is not None:
- x, y, w, h = region
- self.reference_region = region
- screenshot = pyautogui.screenshot(region=region)
- self.reference_image = cv2.cvtColor(np.array(screenshot), cv2.COLOR_RGB2BGR)
- self._display_image(self.reference_label, self.reference_image, "参照物区域")
- self.status_var.set("参照物区域已选择")
-
- def capture_target(self):
- """截取目标区域"""
- self.status_var.set("请在屏幕上框选目标区域...")
- self.root.iconify() # 最小化主窗口
- time.sleep(0.5) # 等待窗口最小化完成
- region = self._capture_screen_region()
- self.root.deiconify() # 恢复主窗口
- self.root.lift() # 提升窗口层级
-
- if region is not None:
- self.target_region = region
- self.status_var.set("目标区域已选择")
- # 立即更新一次目标区域显示
- self._update_target_display()
-
- def _capture_screen_region(self):
- """截取屏幕区域"""
- # 创建全屏透明窗口用于截图
- screenshot_window = tk.Toplevel(self.root)
- screenshot_window.attributes('-fullscreen', True)
- screenshot_window.attributes('-alpha', 0.3)
- screenshot_window.configure(bg='black')
-
- # 初始化截图参数
- start_x = 0
- start_y = 0
- end_x = 0
- end_y = 0
- is_drawing = False
-
- # 创建画布
- canvas = tk.Canvas(screenshot_window, bg='black', highlightthickness=0)
- canvas.pack(fill=BOTH, expand=True)
-
- # 绘制矩形
- rect_id = None
-
- def on_mouse_down(event):
- nonlocal start_x, start_y, is_drawing
- start_x = event.x
- start_y = event.y
- is_drawing = True
-
- def on_mouse_move(event):
- nonlocal end_x, end_y, rect_id
- end_x = event.x
- end_y = event.y
-
- if is_drawing:
- if rect_id:
- canvas.delete(rect_id)
- x1, y1 = min(start_x, end_x), min(start_y, end_y)
- x2, y2 = max(start_x, end_x), max(start_y, end_y)
- rect_id = canvas.create_rectangle(x1, y1, x2, y2, outline='red', width=2)
-
- def on_mouse_up(event):
- nonlocal end_x, end_y, is_drawing
- end_x = event.x
- end_y = event.y
- is_drawing = False
- screenshot_window.destroy()
-
- def cancel_screenshot(event):
- nonlocal start_x, start_y, end_x, end_y
- start_x = 0
- start_y = 0
- end_x = 0
- end_y = 0
- screenshot_window.destroy()
-
- # 绑定事件
- canvas.bind("<Button-1>", on_mouse_down)
- canvas.bind("<B1-Motion>", on_mouse_move)
- canvas.bind("<ButtonRelease-1>", on_mouse_up)
- screenshot_window.bind("<Escape>", cancel_screenshot)
-
- # 等待窗口关闭
- self.root.wait_window(screenshot_window)
-
- # 检查是否有有效的截图区域
- if start_x != end_x and start_y != end_y:
- x1, y1 = min(start_x, end_x), min(start_y, end_y)
- x2, y2 = max(start_x, end_x), max(start_y, end_y)
- return (x1, y1, x2 - x1, y2 - y1)
- else:
- return None
-
- def _display_image(self, label, image, default_text):
- """在标签上显示图像,固定大小为300x200"""
- if image is not None:
- # 调整图像大小为300x200
- image = cv2.resize(image, (300, 200))
-
- # 转换为PIL格式
- image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
- pil_image = Image.fromarray(image_rgb)
- photo = ImageTk.PhotoImage(image=pil_image)
-
- # 更新标签
- label.config(image=photo)
- label.image = photo
- else:
- label.config(text=default_text, image="")
-
- def _update_target_display(self):
- """更新目标区域显示"""
- if self.target_region is not None:
- # 更新窗口位置信息
- self.update_window_rect()
-
- x, y, w, h = self.target_region
- # 截取目标区域
- screenshot = pyautogui.screenshot(region=(x, y, w, h))
- self.target_image = cv2.cvtColor(np.array(screenshot), cv2.COLOR_RGB2BGR)
-
- # 处理目标图像:排除软件窗口和参照物区域
- processed_image = self.target_image.copy()
- if self._is_full_screen(self.target_region):
- processed_image = self._exclude_window_from_image(processed_image)
- if self.reference_region and self._regions_overlap(self.target_region, self.reference_region):
- processed_image = self._exclude_reference_from_image(processed_image)
-
- self._display_image(self.target_label, processed_image, "目标区域")
-
- # 如果正在对比,添加匹配标记
- if self.is_comparing and self.reference_image is not None:
- # 使用原始图像进行对比
- result = self._compare_images(self.reference_image, self.target_image)
- if result is not None:
- similarity, (rx, ry) = result
- h, w = self.reference_image.shape[:2]
-
- # 在显示的图像上绘制矩形标记
- marked_image = processed_image.copy()
- cv2.rectangle(marked_image, (rx - w // 2, ry - h // 2), (rx + w // 2, ry + h // 2), (0, 255, 0), 2)
- self._display_image(self.target_label, marked_image, "目标区域")
-
- def _is_full_screen(self, region):
- """判断区域是否是全屏"""
- screen_width, screen_height = pyautogui.size()
- x, y, w, h = region
- return (w >= screen_width - 10 and h >= screen_height - 10)
-
- def _regions_overlap(self, region1, region2):
- """检查两个区域是否重叠"""
- if not region1 or not region2:
- return False
-
- x1, y1, w1, h1 = region1
- x2, y2, w2, h2 = region2
-
- return not (x1 + w1 < x2 or x2 + w2 < x1 or y1 + h1 < y2 or y2 + h2 < y1)
-
- def _exclude_reference_from_image(self, image):
- """从图像中排除参照物区域"""
- if self.reference_region and self.target_region:
- rx, ry, rw, rh = self.reference_region
- tx, ty, tw, th = self.target_region
-
- # 计算参照物区域在目标区域内的相对位置
- rel_rx = max(0, rx - tx)
- rel_ry = max(0, ry - ty)
- rel_rw = min(rw, tx + tw - rx)
- rel_rh = min(rh, ty + th - ry)
-
- # 如果参照物区域在目标区域内
- if rel_rw > 0 and rel_rh > 0:
- # 创建一个掩码
- mask = np.ones_like(image) * 255
- mask[rel_ry:rel_ry + rel_rh, rel_rx:rel_rx + rel_rw] = 0
-
- # 应用掩码
- result = cv2.bitwise_and(image, mask)
- return result
-
- return image.copy()
-
- def _exclude_window_from_image(self, image):
- """从图像中排除软件窗口区域"""
- if self.window_rect:
- wx, wy, wx2, wy2 = self.window_rect
- ww = wx2 - wx
- wh = wy2 - wy
-
- tx, ty, tw, th = self.target_region
-
- # 计算窗口在目标区域内的相对位置
- rel_wx = max(0, wx - tx)
- rel_wy = max(0, wy - ty)
- rel_wx2 = min(tw, wx2 - tx)
- rel_wy2 = min(th, wy2 - ty)
-
- # 如果窗口在目标区域内
- if rel_wx < rel_wx2 and rel_wy < rel_wy2:
- # 创建一个掩码
- mask = np.ones_like(image) * 255
- mask[rel_wy:rel_wy2, rel_wx:rel_wx2] = 0
-
- # 应用掩码
- result = cv2.bitwise_and(image, mask)
- return result
-
- return image.copy()
-
- def toggle_comparison(self):
- """切换对比状态"""
- if not self.is_comparing:
- # 检查是否已选择图像
- if self.reference_image is None or self.target_region is None:
- messagebox.showerror("错误", "请先选择参照物区域和目标区域")
- return
-
- # 开始对比
- self.is_comparing = True
- self.compare_button.config(text="停止对比 (F8)", bootstyle=DANGER)
- self.status_var.set("正在对比...")
-
- # 启动对比线程
- self.stop_event.clear()
- self.comparison_thread = threading.Thread(target=self._compare_images_continuously)
- self.comparison_thread.daemon = True
- self.comparison_thread.start()
- else:
- # 停止对比
- self.is_comparing = False
- self.compare_button.config(text="开始对比 (F8)", bootstyle=PRIMARY)
- self.status_var.set("已停止对比")
- self.stop_event.set()
-
- def _compare_images_continuously(self):
- """持续对比图像"""
- last_trigger_time = 0
- triggered_once = False
-
- while not self.stop_event.is_set():
- try:
- # 更新窗口位置信息
- self.update_window_rect()
-
- # 截取当前目标区域
- if self.target_region is not None:
- x, y, w, h = self.target_region
- current_target = pyautogui.screenshot(region=(x, y, w, h))
- self.target_image = cv2.cvtColor(np.array(current_target), cv2.COLOR_RGB2BGR)
-
- # 更新目标区域显示
- self.root.after(0, lambda: self._update_target_display())
-
- # 执行图像对比
- result = self._compare_images(self.reference_image, self.target_image)
-
- if result is not None:
- similarity, location = result
-
- # 更新状态
- self.status_var.set(f"找到匹配项,相似度: {similarity:.2f}")
-
- # 检查触发条件
- current_time = time.time()
- trigger_delay = 10 # 10秒延迟
-
- # 触发一次
- if self.trigger_mode.get() == "触发一次" and not triggered_once:
- self._perform_mouse_action(location)
- triggered_once = True
-
- # 无限触发
- elif self.trigger_mode.get() == "无限触发":
- self._perform_mouse_action(location)
-
- # 触发一次过10秒后再次检测
- elif self.trigger_mode.get() == "触发一次过10秒后再次检测":
- if not triggered_once:
- self._perform_mouse_action(location)
- triggered_once = True
- last_trigger_time = current_time
- elif current_time - last_trigger_time >= trigger_delay:
- self._perform_mouse_action(location)
- last_trigger_time = current_time
-
- else:
- self.status_var.set("兄弟丫我正在用吃奶的劲比对中......")
-
- # 控制循环频率
- time.sleep(0.5)
-
- except Exception as e:
- self.status_var.set(f"对比过程中出错: {str(e)}")
- time.sleep(1)
-
- def _compare_images(self, reference, target):
- """对比两个图像,返回相似度和位置"""
- try:
- # 确保两个图像都是灰度图
- if len(reference.shape) == 3:
- reference_gray = cv2.cvtColor(reference, cv2.COLOR_BGR2GRAY)
- else:
- reference_gray = reference
-
- if len(target.shape) == 3:
- target_gray = cv2.cvtColor(target, cv2.COLOR_BGR2GRAY)
- else:
- target_gray = target
-
- # 使用模板匹配
- result = cv2.matchTemplate(target_gray, reference_gray, cv2.TM_CCOEFF_NORMED)
- min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
-
- # 如果相似度超过阈值,返回位置
- if max_val >= self.threshold.get():
- h, w = reference_gray.shape
- center_x = max_loc[0] + w // 2
- center_y = max_loc[1] + h // 2
- return max_val, (center_x, center_y)
-
- return None
-
- except Exception as e:
- self.status_var.set(f"图像对比出错: {str(e)}")
- return None
-
- def _perform_mouse_action(self, location):
- """执行鼠标动作"""
- x, y = location
-
- # 计算屏幕上的实际坐标
- if self.target_region is not None:
- target_x = self.target_region[0] + x
- target_y = self.target_region[1] + y
-
- # 执行鼠标动作
- try:
- # 隐藏窗口以避免干扰
- self.root.iconify()
- time.sleep(0.1)
-
- pyautogui.moveTo(target_x, target_y, duration=0.2)
-
- if self.mouse_action.get() == "单击":
- pyautogui.click()
- elif self.mouse_action.get() == "双击":
- pyautogui.doubleClick()
- elif self.mouse_action.get() == "右击":
- pyautogui.rightClick()
-
- self.status_var.set(f"已执行{self.mouse_action.get()}动作")
-
- # 恢复窗口
- time.sleep(0.1)
- self.root.deiconify()
- self.root.lift()
-
- except Exception as e:
- self.status_var.set(f"鼠标操作出错: {str(e)}")
- # 确保窗口恢复
- self.root.deiconify()
- self.root.lift()
-
- def clear_all(self):
- """清空所有选择和结果"""
- self.reference_image = None
- self.target_image = None
- self.reference_region = None
- self.target_region = None
- self.is_comparing = False
- if self.comparison_thread and self.comparison_thread.is_alive():
- self.stop_event.set()
- self.comparison_thread.join(timeout=1.0)
-
- self.compare_button.config(text="开始对比 (F8)", bootstyle=PRIMARY)
-
- # 重置显示
- self._display_image(self.reference_label, None, "未选择参照物区域")
- self._display_image(self.target_label, None, "未选择目标区域")
-
- self.status_var.set("就绪")
- self.threshold.set(0.8)
- if __name__ == "__main__":
- root = ttkb.Window(themename="flatly")
- app = ScreenCaptureApp(root)
- root.mainloop()