import cv2
import numpy as np
from PIL import ImageGrab
import pytesseract
import time
def detect_major_frames(gray_img):
"""
Etsii ruudun pC$C$alueet (kuten vasen navigaatio ja oikea pC$C$ikkuna).
"""
# 1. EtsitC$C$n reunat
edges = cv2.Canny(gray_img, 30, 100)
# 2. "Sulatus" (Morphological Close)
# KC$ytetC$C$n massiivista kerneliC$ (esim. 100x100). TC$mC$ kadottaa
kaiken tekstin
# ja yksityiskohdat, jC$ttC$en jC$ljelle vain suuret, yhtenC$iset
UI-lohkot.
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (100, 100))
closed = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel)
# EtsitC$C$n nC$iden isojen lohkojen C$C$riviivat
contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
frames = []
screen_area = gray_img.shape[0] * gray_img.shape[1]
for c in contours:
x, y, w, h = cv2.boundingRect(c)
area = w * h
# HylC$tC$C$n alueet, jotka ovat liian pieniC$ (alle 5% ruudusta)
# tai peittC$vC$t koko ruudun (100%)
if area > (screen_area * 0.05) and area < (screen_area * 0.95):
frames.append({"x": x, "y": y, "w": w, "h": h})
# JC$rjestetC$C$n lohkot vasemmalta oikealle
frames.sort(key=lambda f: f["x"])
return frames
def classify_frame(frame, screen_width):
"""
Luokittelee alueen yksinkertaisella sC$C$nnC6llC$.
"""
x, w = frame["x"], frame["w"]
# Jos alue alkaa vasemmasta reunasta ja sen leveys on alle puolet ruudusta
if x < (screen_width * 0.2) and w < (screen_width * 0.5):
return "Sidebar / Navigation List"
# Jos alue on leveC$ ja sijaitsee oikealla
elif w > (screen_width * 0.4):
return "Main Content / Active View"
else:
return "Widget / Dialog"
def generate_ast_for_region(gray_img, frame, frame_id):
"""
Ajaa tutun mikro-analyysin rajatulle alueelle.
"""
fx, fy, fw, fh = frame["x"], frame["y"], frame["w"], frame["h"]
# Rajataan kuva (Crop)
roi_gray = gray_img[fy:fy+fh, fx:fx+fw]
edges = cv2.Canny(roi_gray, 50, 150)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 5))
dilated = cv2.dilate(edges, kernel, iterations=1)
contours, _ = cv2.findContours(dilated, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
nodes = []
for i, contour in enumerate(contours):
x, y, w, h = cv2.boundingRect(contour)
if w < 12 or h < 12 or w > fw * 0.9:
continue
crop = roi_gray[y:y+h, x:x+w]
text = pytesseract.image_to_string(crop, config='--psm 7 -l
eng+fin').strip()
text = "".join([c for c in text if c.isalnum() or c in " .,?!-()/:"])
# LISCTCCN OFFSET: AST-koordinaatit pidetC$C$n absoluuttisina
suhteessa koko nC$yttC6C6n
abs_x, abs_y = fx + x, fy + y
if len(text) > 1:
nodes.append({"id": f"{frame_id}_n{i}", "type": "TextNode",
"content": text, "x": abs_x, "y": abs_y, "w": w, "h": h})
else:
nodes.append({"id": f"{frame_id}_n{i}", "type": "IconNode",
"content": "Graphic", "x": abs_x, "y": abs_y, "w": w, "h": h})
nodes.sort(key=lambda n: (n['y'] // 20, n['x']))
return nodes
def build_hierarchical_ast(gray_img):
print("[*] 1. EtsitC$C$n UI:n pC$C$alueet (Macro Layout)...")
screen_h, screen_w = gray_img.shape
frames = detect_major_frames(gray_img)
print(f"[*] LC6ydettiin {len(frames)} suurta pC$C$aluetta.")
hierarchy = []
for i, frame in enumerate(frames):
frame_type = classify_frame(frame, screen_w)
print(f"[*] 2. Analysoidaan aluetta {i+1}: {frame_type}...")
nodes = generate_ast_for_region(gray_img, frame, f"f{i}")
hierarchy.append({
"frame_id": f"Frame_{i+1}",
"classification": frame_type,
"bounds": frame,
"nodes": nodes
})
return hierarchy
def format_markdown(hierarchy):
md = "# AST-MD: Hierarkkinen KC$yttC6liittymC$kartta
"
for section in hierarchy:
cls = section["classification"]
fid = section["frame_id"]
fb = section["bounds"]
md += f"## {cls} ({fid})
"
md += f"> Alueen rajat: X:{fb['x']}, Y:{fb['y']}, W:{fb['w']},
H:{fb['h']}
"
for n in section["nodes"]:
if n['type'] == 'TextNode':
md += f" - `[{n['id']}]` **Text**: \"{n['content']}\"
(X:{n['x']}, Y:{n['y']})
"
else:
md += f" - `[{n['id']}]` **Icon**: *{n['content']}*
(X:{n['x']}, Y:{n['y']})
"
md += "
"