"""Browser automation for JoinQuant cloud backtest runs. JoinQuant's public ``jqdatasdk`` is a data API; cloud strategy upload/run/export is exposed through the web application. This module keeps that automation optional and configurable so UI selector drift does not affect the core test suite. """ from __future__ import annotations import json import re from datetime import datetime, timezone from pathlib import Path from typing import Any from plugins.joinquant.ingest import ingest_joinquant_outputs from plugins.joinquant.reconcile import reconcile_joinquant DEFAULT_LOGIN_URL = "https://www.joinquant.com/user/login/index" def _require_playwright(): try: from playwright.sync_api import TimeoutError as PlaywrightTimeoutError from playwright.sync_api import sync_playwright except ImportError as exc: raise RuntimeError( "Playwright is required for JoinQuant browser automation. Install it " "in this uv environment, then install Chromium:\n" " uv pip install playwright\n" " uv run playwright install chromium" ) from exc return sync_playwright, PlaywrightTimeoutError def default_browser_config(strategy_url: str = "") -> dict[str, Any]: """Return a selector/action template for JoinQuant browser automation.""" return { "strategy_url": strategy_url, "login_url": DEFAULT_LOGIN_URL, "headless": False, "timeout_ms": 120_000, "notes": [ "Fill strategy_url and selectors after inspecting your JoinQuant strategy page.", "Use `joinquant browser-snapshot` to capture HTML/screenshots for selector tuning.", "The action list is declarative and runs in order.", ], "actions": [ {"type": "goto", "url": "{strategy_url}"}, { "type": "paste_text_file", "selector": "textarea, .ace_text-input, .cm-content, .CodeMirror textarea", "path": "{wrapper_path}", "description": "Paste generated wrapper strategy into the code editor.", }, { "type": "set_input_files", "selector": "input[type=file]", "paths": "{target_csvs}", "description": "Upload all aligned daily target CSV files.", "optional": True, }, { "type": "fill", "selector": "input[name=start_date], input[placeholder*=开始]", "text": "{backtest_start_date}", "optional": True, }, { "type": "fill", "selector": "input[name=end_date], input[placeholder*=结束]", "text": "{backtest_end_date}", "optional": True, }, { "type": "click", "selector": "text=/运行回测|开始回测|回测|Run Backtest|Run/i", "description": "Start the JoinQuant backtest.", }, { "type": "wait_for_selector", "selector": "text=/回测完成|运行完成|Backtest complete|Finished/i", "timeout_ms": 600_000, "optional": True, }, { "type": "download", "selector": "text=/导出成交|下载成交|fills|trades/i", "save_as": "{expected_joinquant_csvs.fills}", "optional": True, }, { "type": "download", "selector": "text=/导出持仓|下载持仓|positions/i", "save_as": "{expected_joinquant_csvs.positions}", "optional": True, }, { "type": "download", "selector": "text=/导出收益|下载收益|pnl|收益/i", "save_as": "{expected_joinquant_csvs.pnl}", "optional": True, }, {"type": "screenshot", "path": "{run_artifact_dir}/final.png"}, ], } def write_browser_config_template(path: str | Path, *, strategy_url: str = "") -> Path: """Write a JSON config template for browser automation.""" out_path = Path(path) out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text( json.dumps(default_browser_config(strategy_url), indent=2, ensure_ascii=False) + "\n", encoding="utf-8", ) return out_path def load_json(path: str | Path) -> dict[str, Any]: """Load a JSON object from disk.""" data = json.loads(Path(path).read_text(encoding="utf-8")) if not isinstance(data, dict): raise ValueError(f"Expected JSON object in {path}") return data def _get_dotted(data: dict[str, Any], dotted: str) -> Any: current: Any = data for part in dotted.split("."): if isinstance(current, dict) and part in current: current = current[part] else: raise KeyError(dotted) return current def _manifest_context(manifest: dict[str, Any], artifact_dir: Path, config: dict[str, Any]) -> dict[str, Any]: targets_dir = Path(str(manifest["targets_dir"])) target_csvs = [str(path) for path in sorted(targets_dir.glob("*.csv"))] if not target_csvs: raise ValueError(f"No target CSV files found under {targets_dir}") target_dates = [ datetime.strptime(Path(path).stem, "%Y%m%d").strftime("%Y-%m-%d") for path in target_csvs ] context = dict(manifest) context.update({ "strategy_url": config.get("strategy_url", ""), "target_csvs": target_csvs, "target_csvs_csv": ",".join(target_csvs), "backtest_start_date": config.get("backtest_start_date") or min(target_dates), "backtest_end_date": config.get("backtest_end_date") or max(target_dates), "run_artifact_dir": str(artifact_dir), }) return context _TOKEN_RE = re.compile(r"^\{([^{}]+)\}$") _PARTIAL_TOKEN_RE = re.compile(r"\{([^{}]+)\}") def resolve_template(value: Any, context: dict[str, Any]) -> Any: """Resolve ``{tokens}`` in config values against the manifest context.""" if isinstance(value, str): whole = _TOKEN_RE.match(value) if whole: return _get_dotted(context, whole.group(1)) def replace(match: re.Match[str]) -> str: resolved = _get_dotted(context, match.group(1)) return str(resolved) return _PARTIAL_TOKEN_RE.sub(replace, value) if isinstance(value, list): return [resolve_template(item, context) for item in value] if isinstance(value, dict): return {key: resolve_template(val, context) for key, val in value.items()} return value def save_login_state( *, storage_state: str | Path, login_url: str = DEFAULT_LOGIN_URL, headless: bool = False, wait_seconds: int = 0, ) -> Path: """Open a browser for manual login and save the authenticated state.""" if headless and wait_seconds <= 0: raise ValueError("headless browser-login requires --wait-seconds > 0") sync_playwright, _ = _require_playwright() state_path = Path(storage_state).expanduser() state_path.parent.mkdir(parents=True, exist_ok=True) with sync_playwright() as pw: browser = pw.chromium.launch(headless=headless) context = browser.new_context() page = context.new_page() page.goto(login_url, wait_until="domcontentloaded") if wait_seconds > 0: page.wait_for_timeout(wait_seconds * 1000) else: input("Log in to JoinQuant in the opened browser, then press Enter here to save state...") context.storage_state(path=str(state_path)) browser.close() state_path.chmod(0o600) return state_path def browser_snapshot( *, url: str, storage_state: str | Path, out_dir: str | Path, headless: bool = True, timeout_ms: int = 60_000, ) -> dict[str, Path]: """Save a logged-in page screenshot and HTML for selector discovery.""" sync_playwright, _ = _require_playwright() root = Path(out_dir) root.mkdir(parents=True, exist_ok=True) html_path = root / "page.html" screenshot_path = root / "page.png" with sync_playwright() as pw: browser = pw.chromium.launch(headless=headless) context = browser.new_context(storage_state=str(Path(storage_state).expanduser())) page = context.new_page() page.set_default_timeout(timeout_ms) page.goto(url, wait_until="networkidle") html_path.write_text(page.content(), encoding="utf-8") page.screenshot(path=str(screenshot_path), full_page=True) browser.close() return {"html": html_path, "screenshot": screenshot_path} def _locator(page: Any, selector: str): return page.locator(selector).first() def _action_fail(action: dict[str, Any], exc: Exception) -> None: if action.get("optional", False): return raise exc def _run_action(page: Any, action: dict[str, Any], context: dict[str, Any], timeout_default: int) -> dict[str, Any]: resolved = resolve_template(action, context) kind = resolved["type"] timeout_ms = int(resolved.get("timeout_ms", timeout_default)) record: dict[str, Any] = {"type": kind, "status": "ok"} try: if kind == "goto": page.goto(str(resolved["url"]), wait_until=resolved.get("wait_until", "domcontentloaded"), timeout=timeout_ms) elif kind == "click": _locator(page, str(resolved["selector"])).click(timeout=timeout_ms) elif kind == "fill": _locator(page, str(resolved["selector"])).fill(str(resolved.get("text", "")), timeout=timeout_ms) elif kind == "press": _locator(page, str(resolved["selector"])).press(str(resolved["key"]), timeout=timeout_ms) elif kind == "set_input_files": paths = resolved.get("paths", []) if isinstance(paths, str): paths = [path for path in paths.split(",") if path] _locator(page, str(resolved["selector"])).set_input_files(paths, timeout=timeout_ms) record["n_files"] = len(paths) elif kind == "paste_text_file": text = Path(str(resolved["path"])).read_text(encoding="utf-8") loc = _locator(page, str(resolved["selector"])) loc.click(timeout=timeout_ms) page.keyboard.press("Control+A") page.keyboard.insert_text(text) record["n_chars"] = len(text) elif kind == "evaluate": page.evaluate(str(resolved["script"]), resolved.get("arg")) elif kind == "wait_for_selector": page.wait_for_selector(str(resolved["selector"]), timeout=timeout_ms) elif kind == "wait_for_timeout": page.wait_for_timeout(int(resolved.get("timeout_ms", 1000))) elif kind == "download": save_as = Path(str(resolved["save_as"])) save_as.parent.mkdir(parents=True, exist_ok=True) with page.expect_download(timeout=timeout_ms) as download_info: _locator(page, str(resolved["selector"])).click(timeout=timeout_ms) download = download_info.value download.save_as(str(save_as)) record["path"] = str(save_as) elif kind == "screenshot": path = Path(str(resolved["path"])) path.parent.mkdir(parents=True, exist_ok=True) page.screenshot(path=str(path), full_page=bool(resolved.get("full_page", True))) record["path"] = str(path) else: raise ValueError(f"Unsupported browser action type: {kind}") except Exception as exc: # pragma: no cover - exercised only with Playwright. record["status"] = "skipped" if resolved.get("optional", False) else "failed" record["error"] = str(exc) _action_fail(resolved, exc) return record def run_browser_backtest( *, manifest_path: str | Path, config_path: str | Path, storage_state: str | Path, out_dir: str | Path | None = None, headless: bool | None = None, auto_reconcile: bool = True, ) -> dict[str, Any]: """Run configured browser automation for a JoinQuant backtest.""" sync_playwright, _ = _require_playwright() manifest = load_json(manifest_path) config = load_json(config_path) artifact_dir = Path(out_dir or Path(str(manifest["joinquant_export_dir"])).parent / "browser_run") artifact_dir.mkdir(parents=True, exist_ok=True) context_vars = _manifest_context(manifest, artifact_dir, config) timeout_ms = int(config.get("timeout_ms", 120_000)) actions = config.get("actions") or [] if not actions: raise ValueError("Browser config contains no actions") records: list[dict[str, Any]] = [] with sync_playwright() as pw: browser = pw.chromium.launch( headless=bool(config.get("headless", False) if headless is None else headless) ) context = browser.new_context( storage_state=str(Path(storage_state).expanduser()), accept_downloads=True, ) page = context.new_page() page.set_default_timeout(timeout_ms) for action in actions: records.append(_run_action(page, action, context_vars, timeout_ms)) browser.close() expected = manifest.get("expected_joinquant_csvs", {}) downloaded = { key: str(path) for key, value in expected.items() if (path := Path(str(value))).exists() } reconcile_paths: dict[str, str] = {} if auto_reconcile and {"fills", "positions", "pnl"}.issubset(downloaded): ingested = ingest_joinquant_outputs( portfolio_name=str(manifest["portfolio_name"]), fills_csv=downloaded["fills"], positions_csv=downloaded["positions"], pnl_csv=downloaded["pnl"], out_dir=Path(str(manifest["joinquant_export_dir"])).parent / "ingested", ) reconciled = reconcile_joinquant( portfolio_name=str(manifest["portfolio_name"]), targets_dir=str(manifest["targets_dir"]), our_fills_path=str(manifest["fills_path"]), our_positions_path=str(manifest["positions_path"]), our_pnl_path=str(manifest["pnl_path"]), jq_fills_path=str(ingested["fills"]), jq_positions_path=str(ingested["positions"]), jq_pnl_path=str(ingested["pnl"]), out_dir=Path(str(manifest["joinquant_export_dir"])).parent / "reconcile", ) reconcile_paths = {key: str(value) for key, value in reconciled.items()} report = { "created_at": datetime.now(timezone.utc).isoformat(), "manifest_path": str(manifest_path), "config_path": str(config_path), "storage_state": str(storage_state), "artifact_dir": str(artifact_dir), "actions": records, "downloaded": downloaded, "reconcile_paths": reconcile_paths, } report_path = artifact_dir / "browser_run_report.json" report["report_path"] = str(report_path) report_path.write_text( json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", ) return report