Download Game! Currently 97 players. Last logged in:FonkensembleUrsaStylusSilencer

Blitzer's Blog >> 72595

Back to blogs index
Posted: 20 Sep 2026 16:12 [ permalink ]
import cv2
import numpy as np
from PIL import ImageGrab
import pytesseract
import time

def detect_major_frames(gray_img):
    """
    Etsii ruudun pC$C$alueet (kuten vasen navigaatio ja oikea pC$C$ikkuna).
    """
    # 1. EtsitC$C$n reunat
    edges = cv2.Canny(gray_img, 30, 100)
    
    # 2. "Sulatus" (Morphological Close)
    # KC$ytetC$C$n massiivista kerneliC$ (esim. 100x100). TC$mC$ kadottaa
kaiken tekstin
    # ja yksityiskohdat, jC$ttC$en jC$ljelle vain suuret, yhtenC$iset
UI-lohkot.
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (100, 100))
    closed = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel)
    
    # EtsitC$C$n nC$iden isojen lohkojen C$C$riviivat
    contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
    
    frames = []
    screen_area = gray_img.shape[0] * gray_img.shape[1]
    
    for c in contours:
        x, y, w, h = cv2.boundingRect(c)
        area = w * h
        
        # HylC$tC$C$n alueet, jotka ovat liian pieniC$ (alle 5% ruudusta) 
        # tai peittC$vC$t koko ruudun (100%)
        if area > (screen_area * 0.05) and area < (screen_area * 0.95):
            frames.append({"x": x, "y": y, "w": w, "h": h})
            
    # JC$rjestetC$C$n lohkot vasemmalta oikealle
    frames.sort(key=lambda f: f["x"])
    return frames

def classify_frame(frame, screen_width):
    """
    Luokittelee alueen yksinkertaisella sC$C$nnC6llC$.
    """
    x, w = frame["x"], frame["w"]
    
    # Jos alue alkaa vasemmasta reunasta ja sen leveys on alle puolet ruudusta
    if x < (screen_width * 0.2) and w < (screen_width * 0.5):
        return "Sidebar / Navigation List"
    # Jos alue on leveC$ ja sijaitsee oikealla
    elif w > (screen_width * 0.4):
        return "Main Content / Active View"
    else:
        return "Widget / Dialog"

def generate_ast_for_region(gray_img, frame, frame_id):
    """
    Ajaa tutun mikro-analyysin rajatulle alueelle.
    """
    fx, fy, fw, fh = frame["x"], frame["y"], frame["w"], frame["h"]
    # Rajataan kuva (Crop)
    roi_gray = gray_img[fy:fy+fh, fx:fx+fw]
    
    edges = cv2.Canny(roi_gray, 50, 150)
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10, 5))
    dilated = cv2.dilate(edges, kernel, iterations=1)
    
    contours, _ = cv2.findContours(dilated, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
    
    nodes = []
    for i, contour in enumerate(contours):
        x, y, w, h = cv2.boundingRect(contour)
        
        if w < 12 or h < 12 or w > fw * 0.9:
            continue
            
        crop = roi_gray[y:y+h, x:x+w]
        text = pytesseract.image_to_string(crop, config='--psm 7 -l
eng+fin').strip()
        text = "".join([c for c in text if c.isalnum() or c in " .,?!-()/:"])
        
        # LISCTCCN OFFSET: AST-koordinaatit pidetC$C$n absoluuttisina
suhteessa koko nC$yttC6C6n
        abs_x, abs_y = fx + x, fy + y
        
        if len(text) > 1:
            nodes.append({"id": f"{frame_id}_n{i}", "type": "TextNode",
"content": text, "x": abs_x, "y": abs_y, "w": w, "h": h})
        else:
            nodes.append({"id": f"{frame_id}_n{i}", "type": "IconNode",
"content": "Graphic", "x": abs_x, "y": abs_y, "w": w, "h": h})
            
    nodes.sort(key=lambda n: (n['y'] // 20, n['x']))
    return nodes

def build_hierarchical_ast(gray_img):
    print("[*] 1. EtsitC$C$n UI:n pC$C$alueet (Macro Layout)...")
    screen_h, screen_w = gray_img.shape
    frames = detect_major_frames(gray_img)
    
    print(f"[*] LC6ydettiin {len(frames)} suurta pC$C$aluetta.")
    
    hierarchy = []
    
    for i, frame in enumerate(frames):
        frame_type = classify_frame(frame, screen_w)
        print(f"[*] 2. Analysoidaan aluetta {i+1}: {frame_type}...")
        
        nodes = generate_ast_for_region(gray_img, frame, f"f{i}")
        
        hierarchy.append({
            "frame_id": f"Frame_{i+1}",
            "classification": frame_type,
            "bounds": frame,
            "nodes": nodes
        })
        
    return hierarchy

def format_markdown(hierarchy):
    md = "# AST-MD: Hierarkkinen KC$yttC6liittymC$kartta

"
    
    for section in hierarchy:
        cls = section["classification"]
        fid = section["frame_id"]
        fb = section["bounds"]
        
        md += f"## {cls} ({fid})
"
        md += f"> Alueen rajat: X:{fb['x']}, Y:{fb['y']}, W:{fb['w']},
H:{fb['h']}

"
        
        for n in section["nodes"]:
            if n['type'] == 'TextNode':
                md += f"  - `[{n['id']}]` **Text**: \"{n['content']}\"
(X:{n['x']}, Y:{n['y']})
"
            else:
                md += f"  - `[{n['id']}]` **Icon**: *{n['content']}*
(X:{n['x']}, Y:{n['y']})
"
        md += "
"