"""Reconcile internal simulator output against normalized JoinQuant output.""" from __future__ import annotations from pathlib import Path import numpy as np import pandas as pd from plugins.joinquant.schema import ( JOINQUANT_FILL_COLUMNS, JOINQUANT_PNL_COLUMNS, JOINQUANT_POSITION_COLUMNS, JOINQUANT_TARGET_COLUMNS, RECONCILE_COLUMNS, ) from plugins.joinquant.symbols import to_joinquant_symbol def _date_text(value: object) -> str: if pd.isna(value): return "" return pd.Timestamp(value).strftime("%Y-%m-%d") def _read_parquet(path: str | Path | None) -> pd.DataFrame: if path is None: return pd.DataFrame() return pd.read_parquet(path) def _normalize_common(df: pd.DataFrame, portfolio_name: str) -> pd.DataFrame: out = df.copy() if "date" in out.columns: out["date"] = out["date"].map(_date_text) else: out["date"] = "" if "portfolio_name" not in out.columns: out["portfolio_name"] = portfolio_name out["portfolio_name"] = out["portfolio_name"].fillna(portfolio_name).astype(str) if "symbol_id" in out.columns: out["symbol_id"] = out["symbol_id"].fillna("").astype(str) if "jq_symbol" not in out.columns and "symbol_id" in out.columns: out["jq_symbol"] = out["symbol_id"].map( lambda s: to_joinquant_symbol(s) if s else "" ) elif "jq_symbol" in out.columns: out["jq_symbol"] = out["jq_symbol"].fillna("").astype(str) return out def _numeric(df: pd.DataFrame, column: str, default: float = 0.0) -> pd.Series: if column not in df.columns: return pd.Series([default] * len(df), index=df.index, dtype=float) return pd.to_numeric(df[column], errors="coerce").replace([np.inf, -np.inf], np.nan) def _weighted_price(group: pd.DataFrame, price_col: str, shares_col: str) -> float: prices = pd.to_numeric(group[price_col], errors="coerce") shares = pd.to_numeric(group[shares_col], errors="coerce").abs() valid = prices.notna() & shares.notna() & (shares > 0) if not valid.any(): return np.nan return float(np.average(prices[valid], weights=shares[valid])) def _load_targets(targets_dir: str | Path, portfolio_name: str) -> pd.DataFrame: root = Path(targets_dir) if not root.exists(): return pd.DataFrame(columns=JOINQUANT_TARGET_COLUMNS) files_by_stem: dict[str, Path] = {} for path in sorted(root.glob("*.csv")): files_by_stem[path.stem] = path for path in sorted(root.glob("*.parquet")): files_by_stem[path.stem] = path frames: list[pd.DataFrame] = [] for path in files_by_stem.values(): if path.suffix == ".parquet": frame = pd.read_parquet(path) else: frame = pd.read_csv(path) frames.append(frame) if not frames: return pd.DataFrame(columns=JOINQUANT_TARGET_COLUMNS) targets = pd.concat(frames, ignore_index=True) targets = _normalize_common(targets, portfolio_name) targets = targets[targets["portfolio_name"].astype(str) == portfolio_name] if "target_shares" not in targets.columns: targets["target_shares"] = 0 return targets.reindex(columns=JOINQUANT_TARGET_COLUMNS) def _aggregate_targets(targets: pd.DataFrame, portfolio_name: str) -> pd.DataFrame: if targets.empty: return pd.DataFrame(columns=["date", "portfolio_name", "symbol_id", "jq_symbol", "target_shares"]) targets = _normalize_common(targets, portfolio_name) targets["target_shares"] = _numeric(targets, "target_shares", 0.0) grouped = ( targets.groupby(["date", "portfolio_name", "symbol_id"], as_index=False) .agg(jq_symbol=("jq_symbol", "last"), target_shares=("target_shares", "last")) ) return grouped def _aggregate_our_fills(fills: pd.DataFrame, portfolio_name: str) -> pd.DataFrame: if fills.empty: return pd.DataFrame(columns=[ "date", "portfolio_name", "symbol_id", "our_filled_shares", "our_position_shares", "our_cost", "our_trade_price", "our_blocked", "our_target_shares", ]) fills = _normalize_common(fills, portfolio_name) fills["traded_shares"] = _numeric(fills, "traded_shares", 0.0) fills["realized_shares"] = _numeric(fills, "realized_shares", np.nan) fills["trade_cost"] = _numeric(fills, "trade_cost", 0.0).fillna(0.0) fills["target_shares"] = _numeric(fills, "target_shares", np.nan) fills["blocked"] = _numeric(fills, "blocked", 0.0).fillna(0.0) price_col = next( (col for col in ["trade_price", "fill_price", "execution_price", "price"] if col in fills.columns), None, ) rows: list[dict[str, object]] = [] for key, group in fills.groupby(["date", "portfolio_name", "symbol_id"], sort=False): row = { "date": key[0], "portfolio_name": key[1], "symbol_id": key[2], "our_filled_shares": float(group["traded_shares"].sum()), "our_position_shares": float(group["realized_shares"].dropna().iloc[-1]) if group["realized_shares"].notna().any() else np.nan, "our_cost": float(group["trade_cost"].sum()), "our_trade_price": _weighted_price(group, price_col, "traded_shares") if price_col else np.nan, "our_blocked": int(group["blocked"].max()), "our_target_shares": float(group["target_shares"].dropna().iloc[-1]) if group["target_shares"].notna().any() else np.nan, } rows.append(row) return pd.DataFrame(rows) def _aggregate_our_positions(positions: pd.DataFrame, portfolio_name: str) -> pd.DataFrame: if positions.empty: return pd.DataFrame(columns=[ "date", "portfolio_name", "symbol_id", "jq_symbol", "our_position_fallback", "our_position_price", ]) positions = _normalize_common(positions, portfolio_name) positions["position_shares"] = _numeric(positions, "position_shares", np.nan) positions["price"] = _numeric(positions, "price", np.nan) return ( positions.groupby(["date", "portfolio_name", "symbol_id"], as_index=False) .agg( jq_symbol=("jq_symbol", "last"), our_position_fallback=("position_shares", "last"), our_position_price=("price", "last"), ) ) def _aggregate_jq_fills(fills: pd.DataFrame, portfolio_name: str) -> pd.DataFrame: if fills.empty: return pd.DataFrame(columns=[ "date", "portfolio_name", "symbol_id", "jq_filled_shares", "jq_trade_price", "jq_cost", "jq_blocked", "jq_requested_shares", "raw_status", ]) fills = _normalize_common(fills, portfolio_name) for col in JOINQUANT_FILL_COLUMNS: if col not in fills.columns: fills[col] = np.nan fills["filled_shares"] = _numeric(fills, "filled_shares", 0.0).fillna(0.0) fills["requested_shares"] = _numeric(fills, "requested_shares", np.nan) fills["fill_price"] = _numeric(fills, "fill_price", np.nan) fills["trade_cost"] = _numeric(fills, "trade_cost", 0.0).fillna(0.0) fills["blocked"] = _numeric(fills, "blocked", 0.0).fillna(0.0) fills["raw_status"] = fills["raw_status"].fillna("").astype(str) rows: list[dict[str, object]] = [] for key, group in fills.groupby(["date", "portfolio_name", "symbol_id"], sort=False): row = { "date": key[0], "portfolio_name": key[1], "symbol_id": key[2], "jq_filled_shares": float(group["filled_shares"].sum()), "jq_trade_price": _weighted_price(group, "fill_price", "filled_shares"), "jq_cost": float(group["trade_cost"].sum()), "jq_blocked": int(group["blocked"].max()), "jq_requested_shares": float(group["requested_shares"].dropna().iloc[-1]) if group["requested_shares"].notna().any() else np.nan, "raw_status": ";".join([s for s in group["raw_status"].astype(str) if s]), } rows.append(row) return pd.DataFrame(rows) def _aggregate_jq_positions(positions: pd.DataFrame, portfolio_name: str) -> pd.DataFrame: if positions.empty: return pd.DataFrame(columns=[ "date", "portfolio_name", "symbol_id", "jq_symbol", "jq_position_shares", ]) positions = _normalize_common(positions, portfolio_name) for col in JOINQUANT_POSITION_COLUMNS: if col not in positions.columns: positions[col] = np.nan positions["position_shares"] = _numeric(positions, "position_shares", np.nan) return ( positions.groupby(["date", "portfolio_name", "symbol_id"], as_index=False) .agg(jq_symbol=("jq_symbol", "last"), jq_position_shares=("position_shares", "last")) ) def _portfolio_frame(df: pd.DataFrame, portfolio_name: str, prefix: str) -> pd.DataFrame: if df.empty: return pd.DataFrame(columns=["date", "portfolio_name"]) out = _normalize_common(df, portfolio_name) for col in JOINQUANT_PNL_COLUMNS: if col not in out.columns: out[col] = np.nan keep = ["date", "portfolio_name", "gross_exposure", "net_exposure", "cash", "total_value", "pnl", "cost", "turnover"] out = out[keep].copy() for col in keep[2:]: out[col] = pd.to_numeric(out[col], errors="coerce") out = out.groupby(["date", "portfolio_name"], as_index=False).last() return out.rename(columns={col: f"{prefix}_{col}" for col in keep[2:]}) def _infer_booksize(targets: pd.DataFrame, our_pnl: pd.DataFrame, jq_pnl: pd.DataFrame) -> float: candidates: list[float] = [] if "target_value" in targets.columns: gross = ( pd.to_numeric(targets["target_value"], errors="coerce") .abs() .replace([np.inf, -np.inf], np.nan) .dropna() .sum() ) if gross > 0: candidates.append(float(gross)) for df in (our_pnl, jq_pnl): if "gross_exposure" in df.columns: val = pd.to_numeric(df["gross_exposure"], errors="coerce").max() if pd.notna(val) and val > 0: candidates.append(float(val)) return max(candidates) if candidates else 1.0 def _status_reason(raw_status: object) -> str | None: text = str(raw_status or "").lower() if "suspend" in text or "halt" in text: return "SUSPENSION" if "limit_up" in text or "limit up" in text or "up_limit" in text: return "LIMIT_UP_BLOCK" if "limit_down" in text or "limit down" in text or "down_limit" in text: return "LIMIT_DOWN_BLOCK" if "volume" in text or "liquid" in text: return "VOLUME_OR_LIQUIDITY" if "cash" in text or "margin" in text: return "CASH_CONSTRAINT" return None def _classify_symbol_row( row: pd.Series, *, share_tol: float, price_rel_tol: float, value_tol: float, pnl_tol: float, ) -> str: target = row.get("target_shares", np.nan) filled_diff = abs(row.get("filled_share_diff", 0.0)) position_diff = abs(row.get("position_share_diff", 0.0)) our_present = bool(row.get("_our_present", False)) jq_present = bool(row.get("_jq_present", False)) if pd.notna(target) and target < 0 and (filled_diff > share_tol or position_diff > share_tol or not jq_present): return "SHORT_NOT_SUPPORTED" if not jq_present and (our_present or pd.notna(target)): return "MISSING_IN_JOINQUANT" if not our_present and jq_present: return "MISSING_IN_OUR_SYSTEM" status_reason = _status_reason(row.get("raw_status", "")) if (filled_diff > share_tol or position_diff > share_tol) and status_reason: return status_reason if filled_diff > share_tol or position_diff > share_tol: return "UNKNOWN" our_price = row.get("our_trade_price", np.nan) jq_price = row.get("jq_trade_price", np.nan) if pd.notna(our_price) and pd.notna(jq_price): denom = max(abs(float(our_price)), abs(float(jq_price)), 1.0) if abs(float(our_price) - float(jq_price)) > price_rel_tol * denom: return "PRICE_MISMATCH" if abs(row.get("cost_diff", 0.0)) > value_tol: return "COST_MODEL" if abs(row.get("pnl_diff", 0.0)) > pnl_tol: return "UNKNOWN" return "MATCH" def _classify_portfolio_row(row: pd.Series, value_tol: float, pnl_tol: float) -> str: our_present = bool(row.get("_our_present", False)) jq_present = bool(row.get("_jq_present", False)) if not jq_present and our_present: return "MISSING_IN_JOINQUANT" if not our_present and jq_present: return "MISSING_IN_OUR_SYSTEM" if abs(row.get("cost_diff", 0.0)) > value_tol: return "COST_MODEL" if abs(row.get("pnl_diff", 0.0)) > pnl_tol: return "UNKNOWN" return "MATCH" def _build_symbol_reconcile( *, portfolio_name: str, targets: pd.DataFrame, our_fills: pd.DataFrame, our_positions: pd.DataFrame, our_pnl: pd.DataFrame, jq_fills: pd.DataFrame, jq_positions: pd.DataFrame, jq_pnl: pd.DataFrame, share_tol: float, price_rel_tol: float, value_tol: float, pnl_tol: float, ) -> pd.DataFrame: target_agg = _aggregate_targets(targets, portfolio_name) our_fill_agg = _aggregate_our_fills(our_fills, portfolio_name) our_pos_agg = _aggregate_our_positions(our_positions, portfolio_name) jq_fill_agg = _aggregate_jq_fills(jq_fills, portfolio_name) jq_pos_agg = _aggregate_jq_positions(jq_positions, portfolio_name) key_cols = ["date", "portfolio_name", "symbol_id"] keys = [] for frame in [target_agg, our_fill_agg, our_pos_agg, jq_fill_agg, jq_pos_agg]: if not frame.empty: keys.append(frame[key_cols]) if not keys: return pd.DataFrame(columns=RECONCILE_COLUMNS) base = pd.concat(keys, ignore_index=True).drop_duplicates() result = base.merge(target_agg, on=key_cols, how="left") result = result.merge(our_fill_agg, on=key_cols, how="left") result = result.merge(our_pos_agg, on=key_cols, how="left", suffixes=("", "_ourpos")) result = result.merge(jq_fill_agg, on=key_cols, how="left") result = result.merge(jq_pos_agg, on=key_cols, how="left", suffixes=("", "_jqpos")) jq_symbol_cols = [col for col in result.columns if col.startswith("jq_symbol")] jq_symbol_values = result[jq_symbol_cols].copy() if jq_symbol_cols else pd.DataFrame(index=result.index) result["jq_symbol"] = "" for col in jq_symbol_values.columns: values = jq_symbol_values[col].fillna("").astype(str) result["jq_symbol"] = result["jq_symbol"].mask( result["jq_symbol"].eq("") & values.ne(""), values, ) result["jq_symbol"] = result.apply( lambda row: row["jq_symbol"] or to_joinquant_symbol(row["symbol_id"]), axis=1, ) result["_our_present"] = ( result[["our_filled_shares", "our_position_shares", "our_position_fallback"]] .notna() .any(axis=1) ) result["_jq_present"] = ( result[["jq_filled_shares", "jq_position_shares"]].notna().any(axis=1) ) target_shares = pd.to_numeric(result["target_shares"], errors="coerce") our_target = pd.to_numeric(result["our_target_shares"], errors="coerce") jq_target = pd.to_numeric(result["jq_requested_shares"], errors="coerce") result["target_shares"] = target_shares.where(target_shares.notna(), our_target) result["target_shares"] = result["target_shares"].where( result["target_shares"].notna(), jq_target, ) our_position = pd.to_numeric(result["our_position_shares"], errors="coerce") our_position_fallback = pd.to_numeric(result["our_position_fallback"], errors="coerce") result["our_position_shares"] = our_position.where( our_position.notna(), our_position_fallback, ) for col in [ "target_shares", "our_filled_shares", "jq_filled_shares", "our_position_shares", "jq_position_shares", "our_cost", "jq_cost", ]: result[col] = pd.to_numeric(result[col], errors="coerce").fillna(0.0) result["filled_share_diff"] = result["our_filled_shares"] - result["jq_filled_shares"] result["position_share_diff"] = result["our_position_shares"] - result["jq_position_shares"] result["trade_price_diff"] = np.where( result["our_trade_price"].notna() & result["jq_trade_price"].notna(), result["our_trade_price"] - result["jq_trade_price"], np.nan, ) result["cost_diff"] = result["our_cost"] - result["jq_cost"] our_daily = _portfolio_frame(our_pnl, portfolio_name, "our") jq_daily = _portfolio_frame(jq_pnl, portfolio_name, "jq") pnl_daily = our_daily.merge(jq_daily, on=["date", "portfolio_name"], how="outer") if not pnl_daily.empty: pnl_daily["our_pnl"] = pd.to_numeric(pnl_daily.get("our_pnl"), errors="coerce").fillna(0.0) pnl_daily["jq_pnl"] = pd.to_numeric(pnl_daily.get("jq_pnl"), errors="coerce").fillna(0.0) pnl_daily["pnl_diff"] = pnl_daily["our_pnl"] - pnl_daily["jq_pnl"] result = result.merge( pnl_daily[["date", "portfolio_name", "our_pnl", "jq_pnl", "pnl_diff"]], on=["date", "portfolio_name"], how="left", ) else: result["our_pnl"] = 0.0 result["jq_pnl"] = 0.0 result["pnl_diff"] = 0.0 for col in ["our_pnl", "jq_pnl", "pnl_diff"]: result[col] = pd.to_numeric(result[col], errors="coerce").fillna(0.0) result["raw_status"] = result.get("raw_status", "").fillna("") result["diff_reason"] = result.apply( _classify_symbol_row, axis=1, share_tol=share_tol, price_rel_tol=price_rel_tol, value_tol=value_tol, pnl_tol=pnl_tol, ) return result[RECONCILE_COLUMNS].sort_values( ["date", "portfolio_name", "symbol_id"] ).reset_index(drop=True) def _build_portfolio_summary( *, portfolio_name: str, our_pnl: pd.DataFrame, jq_pnl: pd.DataFrame, value_tol: float, pnl_tol: float, ) -> pd.DataFrame: our = _portfolio_frame(our_pnl, portfolio_name, "our") jq = _portfolio_frame(jq_pnl, portfolio_name, "jq") if our.empty and jq.empty: return pd.DataFrame(columns=[ "date", "portfolio_name", "diff_reason", "our_pnl", "jq_pnl", "pnl_diff", ]) summary = our.merge(jq, on=["date", "portfolio_name"], how="outer") summary["_our_present"] = summary.filter(regex=r"^our_").notna().any(axis=1) summary["_jq_present"] = summary.filter(regex=r"^jq_").notna().any(axis=1) metrics = ["gross_exposure", "net_exposure", "cash", "total_value", "pnl", "cost", "turnover"] for metric in metrics: our_col = f"our_{metric}" jq_col = f"jq_{metric}" if our_col not in summary.columns: summary[our_col] = np.nan if jq_col not in summary.columns: summary[jq_col] = np.nan summary[f"{metric}_diff"] = ( pd.to_numeric(summary[our_col], errors="coerce").fillna(0.0) - pd.to_numeric(summary[jq_col], errors="coerce").fillna(0.0) ) summary["our_cumulative_pnl"] = ( pd.to_numeric(summary["our_pnl"], errors="coerce").fillna(0.0).cumsum() ) summary["jq_cumulative_pnl"] = ( pd.to_numeric(summary["jq_pnl"], errors="coerce").fillna(0.0).cumsum() ) summary["cumulative_pnl_diff"] = summary["our_cumulative_pnl"] - summary["jq_cumulative_pnl"] summary["diff_reason"] = summary.apply( _classify_portfolio_row, axis=1, value_tol=value_tol, pnl_tol=pnl_tol, ) summary = summary.drop(columns=["_our_present", "_jq_present"]) return summary.sort_values(["date", "portfolio_name"]).reset_index(drop=True) def _write_summary_md( path: Path, *, portfolio_name: str, symbol_report: pd.DataFrame, portfolio_summary: pd.DataFrame, ) -> None: symbol_counts = ( symbol_report["diff_reason"].value_counts().sort_index() if not symbol_report.empty else pd.Series(dtype=int) ) portfolio_counts = ( portfolio_summary["diff_reason"].value_counts().sort_index() if not portfolio_summary.empty else pd.Series(dtype=int) ) lines = [ "# JoinQuant Reconciliation Summary", "", f"Portfolio: `{portfolio_name}`", "", "## Per-symbol Difference Counts", "", ] if symbol_counts.empty: lines.append("No per-symbol rows were produced.") else: for reason, count in symbol_counts.items(): lines.append(f"- {reason}: {int(count)}") lines.extend(["", "## Daily Portfolio Difference Counts", ""]) if portfolio_counts.empty: lines.append("No daily portfolio rows were produced.") else: for reason, count in portfolio_counts.items(): lines.append(f"- {reason}: {int(count)}") if not portfolio_summary.empty: lines.extend(["", "## Daily Portfolio Preview", ""]) preview_cols = [ "date", "diff_reason", "our_pnl", "jq_pnl", "pnl_diff", "our_cost", "jq_cost", "cost_diff", ] preview_cols = [col for col in preview_cols if col in portfolio_summary.columns] lines.append(",".join(preview_cols)) for row in portfolio_summary[preview_cols].head(20).itertuples(index=False): lines.append(",".join(str(value) for value in row)) path.write_text("\n".join(lines) + "\n", encoding="utf-8") def reconcile_joinquant( *, portfolio_name: str, targets_dir: str | Path, our_fills_path: str | Path, our_positions_path: str | Path, our_pnl_path: str | Path, jq_fills_path: str | Path, jq_positions_path: str | Path, jq_pnl_path: str | Path, out_dir: str | Path = "plugins_output/joinquant/reconcile", share_tolerance: float = 0.0, price_rel_tolerance: float = 1e-4, pnl_tolerance: float = 1.0, booksize: float | None = None, ) -> dict[str, Path]: """Reconcile JoinQuant output against internal simulator output.""" targets = _load_targets(targets_dir, portfolio_name) our_fills = _read_parquet(our_fills_path) our_positions = _read_parquet(our_positions_path) our_pnl = _read_parquet(our_pnl_path) jq_fills = _read_parquet(jq_fills_path) jq_positions = _read_parquet(jq_positions_path) jq_pnl = _read_parquet(jq_pnl_path) inferred_booksize = booksize or _infer_booksize(targets, our_pnl, jq_pnl) value_tol = max(1.0, 1e-6 * float(inferred_booksize)) symbol_report = _build_symbol_reconcile( portfolio_name=portfolio_name, targets=targets, our_fills=our_fills, our_positions=our_positions, our_pnl=our_pnl, jq_fills=jq_fills, jq_positions=jq_positions, jq_pnl=jq_pnl, share_tol=share_tolerance, price_rel_tol=price_rel_tolerance, value_tol=value_tol, pnl_tol=pnl_tolerance, ) portfolio_summary = _build_portfolio_summary( portfolio_name=portfolio_name, our_pnl=our_pnl, jq_pnl=jq_pnl, value_tol=value_tol, pnl_tol=pnl_tolerance, ) root = Path(out_dir) / portfolio_name root.mkdir(parents=True, exist_ok=True) paths = { "daily_reconcile": root / "daily_reconcile.pq", "summary_csv": root / "summary.csv", "summary_md": root / "summary.md", } symbol_report.to_parquet(paths["daily_reconcile"], index=False) portfolio_summary.to_csv(paths["summary_csv"], index=False) _write_summary_md( paths["summary_md"], portfolio_name=portfolio_name, symbol_report=symbol_report, portfolio_summary=portfolio_summary, ) return paths