Yuxuan Yan
608 lines
23 KiB
Python
608 lines
23 KiB
Python
"""Reconcile internal simulator output against normalized JoinQuant output."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
from plugins.joinquant.schema import (
|
|
JOINQUANT_FILL_COLUMNS,
|
|
JOINQUANT_PNL_COLUMNS,
|
|
JOINQUANT_POSITION_COLUMNS,
|
|
JOINQUANT_TARGET_COLUMNS,
|
|
RECONCILE_COLUMNS,
|
|
)
|
|
from plugins.joinquant.symbols import to_joinquant_symbol
|
|
|
|
|
|
def _date_text(value: object) -> str:
|
|
if pd.isna(value):
|
|
return ""
|
|
return pd.Timestamp(value).strftime("%Y-%m-%d")
|
|
|
|
|
|
def _read_parquet(path: str | Path | None) -> pd.DataFrame:
|
|
if path is None:
|
|
return pd.DataFrame()
|
|
return pd.read_parquet(path)
|
|
|
|
|
|
def _normalize_common(df: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
|
out = df.copy()
|
|
if "date" in out.columns:
|
|
out["date"] = out["date"].map(_date_text)
|
|
else:
|
|
out["date"] = ""
|
|
if "portfolio_name" not in out.columns:
|
|
out["portfolio_name"] = portfolio_name
|
|
out["portfolio_name"] = out["portfolio_name"].fillna(portfolio_name).astype(str)
|
|
if "symbol_id" in out.columns:
|
|
out["symbol_id"] = out["symbol_id"].fillna("").astype(str)
|
|
if "jq_symbol" not in out.columns and "symbol_id" in out.columns:
|
|
out["jq_symbol"] = out["symbol_id"].map(
|
|
lambda s: to_joinquant_symbol(s) if s else ""
|
|
)
|
|
elif "jq_symbol" in out.columns:
|
|
out["jq_symbol"] = out["jq_symbol"].fillna("").astype(str)
|
|
return out
|
|
|
|
|
|
def _numeric(df: pd.DataFrame, column: str, default: float = 0.0) -> pd.Series:
|
|
if column not in df.columns:
|
|
return pd.Series([default] * len(df), index=df.index, dtype=float)
|
|
return pd.to_numeric(df[column], errors="coerce").replace([np.inf, -np.inf], np.nan)
|
|
|
|
|
|
def _weighted_price(group: pd.DataFrame, price_col: str, shares_col: str) -> float:
|
|
prices = pd.to_numeric(group[price_col], errors="coerce")
|
|
shares = pd.to_numeric(group[shares_col], errors="coerce").abs()
|
|
valid = prices.notna() & shares.notna() & (shares > 0)
|
|
if not valid.any():
|
|
return np.nan
|
|
return float(np.average(prices[valid], weights=shares[valid]))
|
|
|
|
|
|
def _load_targets(targets_dir: str | Path, portfolio_name: str) -> pd.DataFrame:
|
|
root = Path(targets_dir)
|
|
if not root.exists():
|
|
return pd.DataFrame(columns=JOINQUANT_TARGET_COLUMNS)
|
|
|
|
files_by_stem: dict[str, Path] = {}
|
|
for path in sorted(root.glob("*.csv")):
|
|
files_by_stem[path.stem] = path
|
|
for path in sorted(root.glob("*.parquet")):
|
|
files_by_stem[path.stem] = path
|
|
|
|
frames: list[pd.DataFrame] = []
|
|
for path in files_by_stem.values():
|
|
if path.suffix == ".parquet":
|
|
frame = pd.read_parquet(path)
|
|
else:
|
|
frame = pd.read_csv(path)
|
|
frames.append(frame)
|
|
|
|
if not frames:
|
|
return pd.DataFrame(columns=JOINQUANT_TARGET_COLUMNS)
|
|
targets = pd.concat(frames, ignore_index=True)
|
|
targets = _normalize_common(targets, portfolio_name)
|
|
targets = targets[targets["portfolio_name"].astype(str) == portfolio_name]
|
|
if "target_shares" not in targets.columns:
|
|
targets["target_shares"] = 0
|
|
return targets.reindex(columns=JOINQUANT_TARGET_COLUMNS)
|
|
|
|
|
|
def _aggregate_targets(targets: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
|
if targets.empty:
|
|
return pd.DataFrame(columns=["date", "portfolio_name", "symbol_id", "jq_symbol", "target_shares"])
|
|
targets = _normalize_common(targets, portfolio_name)
|
|
targets["target_shares"] = _numeric(targets, "target_shares", 0.0)
|
|
grouped = (
|
|
targets.groupby(["date", "portfolio_name", "symbol_id"], as_index=False)
|
|
.agg(jq_symbol=("jq_symbol", "last"), target_shares=("target_shares", "last"))
|
|
)
|
|
return grouped
|
|
|
|
|
|
def _aggregate_our_fills(fills: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
|
if fills.empty:
|
|
return pd.DataFrame(columns=[
|
|
"date", "portfolio_name", "symbol_id", "our_filled_shares",
|
|
"our_position_shares", "our_cost", "our_trade_price", "our_blocked",
|
|
"our_target_shares",
|
|
])
|
|
fills = _normalize_common(fills, portfolio_name)
|
|
fills["traded_shares"] = _numeric(fills, "traded_shares", 0.0)
|
|
fills["realized_shares"] = _numeric(fills, "realized_shares", np.nan)
|
|
fills["trade_cost"] = _numeric(fills, "trade_cost", 0.0).fillna(0.0)
|
|
fills["target_shares"] = _numeric(fills, "target_shares", np.nan)
|
|
fills["blocked"] = _numeric(fills, "blocked", 0.0).fillna(0.0)
|
|
price_col = next(
|
|
(col for col in ["trade_price", "fill_price", "execution_price", "price"] if col in fills.columns),
|
|
None,
|
|
)
|
|
|
|
rows: list[dict[str, object]] = []
|
|
for key, group in fills.groupby(["date", "portfolio_name", "symbol_id"], sort=False):
|
|
row = {
|
|
"date": key[0],
|
|
"portfolio_name": key[1],
|
|
"symbol_id": key[2],
|
|
"our_filled_shares": float(group["traded_shares"].sum()),
|
|
"our_position_shares": float(group["realized_shares"].dropna().iloc[-1])
|
|
if group["realized_shares"].notna().any() else np.nan,
|
|
"our_cost": float(group["trade_cost"].sum()),
|
|
"our_trade_price": _weighted_price(group, price_col, "traded_shares")
|
|
if price_col else np.nan,
|
|
"our_blocked": int(group["blocked"].max()),
|
|
"our_target_shares": float(group["target_shares"].dropna().iloc[-1])
|
|
if group["target_shares"].notna().any() else np.nan,
|
|
}
|
|
rows.append(row)
|
|
return pd.DataFrame(rows)
|
|
|
|
|
|
def _aggregate_our_positions(positions: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
|
if positions.empty:
|
|
return pd.DataFrame(columns=[
|
|
"date", "portfolio_name", "symbol_id", "jq_symbol",
|
|
"our_position_fallback", "our_position_price",
|
|
])
|
|
positions = _normalize_common(positions, portfolio_name)
|
|
positions["position_shares"] = _numeric(positions, "position_shares", np.nan)
|
|
positions["price"] = _numeric(positions, "price", np.nan)
|
|
return (
|
|
positions.groupby(["date", "portfolio_name", "symbol_id"], as_index=False)
|
|
.agg(
|
|
jq_symbol=("jq_symbol", "last"),
|
|
our_position_fallback=("position_shares", "last"),
|
|
our_position_price=("price", "last"),
|
|
)
|
|
)
|
|
|
|
|
|
def _aggregate_jq_fills(fills: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
|
if fills.empty:
|
|
return pd.DataFrame(columns=[
|
|
"date", "portfolio_name", "symbol_id", "jq_filled_shares",
|
|
"jq_trade_price", "jq_cost", "jq_blocked", "jq_requested_shares",
|
|
"raw_status",
|
|
])
|
|
fills = _normalize_common(fills, portfolio_name)
|
|
for col in JOINQUANT_FILL_COLUMNS:
|
|
if col not in fills.columns:
|
|
fills[col] = np.nan
|
|
fills["filled_shares"] = _numeric(fills, "filled_shares", 0.0).fillna(0.0)
|
|
fills["requested_shares"] = _numeric(fills, "requested_shares", np.nan)
|
|
fills["fill_price"] = _numeric(fills, "fill_price", np.nan)
|
|
fills["trade_cost"] = _numeric(fills, "trade_cost", 0.0).fillna(0.0)
|
|
fills["blocked"] = _numeric(fills, "blocked", 0.0).fillna(0.0)
|
|
fills["raw_status"] = fills["raw_status"].fillna("").astype(str)
|
|
|
|
rows: list[dict[str, object]] = []
|
|
for key, group in fills.groupby(["date", "portfolio_name", "symbol_id"], sort=False):
|
|
row = {
|
|
"date": key[0],
|
|
"portfolio_name": key[1],
|
|
"symbol_id": key[2],
|
|
"jq_filled_shares": float(group["filled_shares"].sum()),
|
|
"jq_trade_price": _weighted_price(group, "fill_price", "filled_shares"),
|
|
"jq_cost": float(group["trade_cost"].sum()),
|
|
"jq_blocked": int(group["blocked"].max()),
|
|
"jq_requested_shares": float(group["requested_shares"].dropna().iloc[-1])
|
|
if group["requested_shares"].notna().any() else np.nan,
|
|
"raw_status": ";".join([s for s in group["raw_status"].astype(str) if s]),
|
|
}
|
|
rows.append(row)
|
|
return pd.DataFrame(rows)
|
|
|
|
|
|
def _aggregate_jq_positions(positions: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
|
if positions.empty:
|
|
return pd.DataFrame(columns=[
|
|
"date", "portfolio_name", "symbol_id", "jq_symbol", "jq_position_shares",
|
|
])
|
|
positions = _normalize_common(positions, portfolio_name)
|
|
for col in JOINQUANT_POSITION_COLUMNS:
|
|
if col not in positions.columns:
|
|
positions[col] = np.nan
|
|
positions["position_shares"] = _numeric(positions, "position_shares", np.nan)
|
|
return (
|
|
positions.groupby(["date", "portfolio_name", "symbol_id"], as_index=False)
|
|
.agg(jq_symbol=("jq_symbol", "last"), jq_position_shares=("position_shares", "last"))
|
|
)
|
|
|
|
|
|
def _portfolio_frame(df: pd.DataFrame, portfolio_name: str, prefix: str) -> pd.DataFrame:
|
|
if df.empty:
|
|
return pd.DataFrame(columns=["date", "portfolio_name"])
|
|
out = _normalize_common(df, portfolio_name)
|
|
for col in JOINQUANT_PNL_COLUMNS:
|
|
if col not in out.columns:
|
|
out[col] = np.nan
|
|
keep = ["date", "portfolio_name", "gross_exposure", "net_exposure", "cash", "total_value", "pnl", "cost", "turnover"]
|
|
out = out[keep].copy()
|
|
for col in keep[2:]:
|
|
out[col] = pd.to_numeric(out[col], errors="coerce")
|
|
out = out.groupby(["date", "portfolio_name"], as_index=False).last()
|
|
return out.rename(columns={col: f"{prefix}_{col}" for col in keep[2:]})
|
|
|
|
|
|
def _infer_booksize(targets: pd.DataFrame, our_pnl: pd.DataFrame, jq_pnl: pd.DataFrame) -> float:
|
|
candidates: list[float] = []
|
|
if "target_value" in targets.columns:
|
|
gross = (
|
|
pd.to_numeric(targets["target_value"], errors="coerce")
|
|
.abs()
|
|
.replace([np.inf, -np.inf], np.nan)
|
|
.dropna()
|
|
.sum()
|
|
)
|
|
if gross > 0:
|
|
candidates.append(float(gross))
|
|
for df in (our_pnl, jq_pnl):
|
|
if "gross_exposure" in df.columns:
|
|
val = pd.to_numeric(df["gross_exposure"], errors="coerce").max()
|
|
if pd.notna(val) and val > 0:
|
|
candidates.append(float(val))
|
|
return max(candidates) if candidates else 1.0
|
|
|
|
|
|
def _status_reason(raw_status: object) -> str | None:
|
|
text = str(raw_status or "").lower()
|
|
if "suspend" in text or "halt" in text:
|
|
return "SUSPENSION"
|
|
if "limit_up" in text or "limit up" in text or "up_limit" in text:
|
|
return "LIMIT_UP_BLOCK"
|
|
if "limit_down" in text or "limit down" in text or "down_limit" in text:
|
|
return "LIMIT_DOWN_BLOCK"
|
|
if "volume" in text or "liquid" in text:
|
|
return "VOLUME_OR_LIQUIDITY"
|
|
if "cash" in text or "margin" in text:
|
|
return "CASH_CONSTRAINT"
|
|
return None
|
|
|
|
|
|
def _classify_symbol_row(
|
|
row: pd.Series,
|
|
*,
|
|
share_tol: float,
|
|
price_rel_tol: float,
|
|
value_tol: float,
|
|
pnl_tol: float,
|
|
) -> str:
|
|
target = row.get("target_shares", np.nan)
|
|
filled_diff = abs(row.get("filled_share_diff", 0.0))
|
|
position_diff = abs(row.get("position_share_diff", 0.0))
|
|
our_present = bool(row.get("_our_present", False))
|
|
jq_present = bool(row.get("_jq_present", False))
|
|
|
|
if pd.notna(target) and target < 0 and (filled_diff > share_tol or position_diff > share_tol or not jq_present):
|
|
return "SHORT_NOT_SUPPORTED"
|
|
if not jq_present and (our_present or pd.notna(target)):
|
|
return "MISSING_IN_JOINQUANT"
|
|
if not our_present and jq_present:
|
|
return "MISSING_IN_OUR_SYSTEM"
|
|
|
|
status_reason = _status_reason(row.get("raw_status", ""))
|
|
if (filled_diff > share_tol or position_diff > share_tol) and status_reason:
|
|
return status_reason
|
|
if filled_diff > share_tol or position_diff > share_tol:
|
|
return "UNKNOWN"
|
|
|
|
our_price = row.get("our_trade_price", np.nan)
|
|
jq_price = row.get("jq_trade_price", np.nan)
|
|
if pd.notna(our_price) and pd.notna(jq_price):
|
|
denom = max(abs(float(our_price)), abs(float(jq_price)), 1.0)
|
|
if abs(float(our_price) - float(jq_price)) > price_rel_tol * denom:
|
|
return "PRICE_MISMATCH"
|
|
|
|
if abs(row.get("cost_diff", 0.0)) > value_tol:
|
|
return "COST_MODEL"
|
|
if abs(row.get("pnl_diff", 0.0)) > pnl_tol:
|
|
return "UNKNOWN"
|
|
return "MATCH"
|
|
|
|
|
|
def _classify_portfolio_row(row: pd.Series, value_tol: float, pnl_tol: float) -> str:
|
|
our_present = bool(row.get("_our_present", False))
|
|
jq_present = bool(row.get("_jq_present", False))
|
|
if not jq_present and our_present:
|
|
return "MISSING_IN_JOINQUANT"
|
|
if not our_present and jq_present:
|
|
return "MISSING_IN_OUR_SYSTEM"
|
|
if abs(row.get("cost_diff", 0.0)) > value_tol:
|
|
return "COST_MODEL"
|
|
if abs(row.get("pnl_diff", 0.0)) > pnl_tol:
|
|
return "UNKNOWN"
|
|
return "MATCH"
|
|
|
|
|
|
def _build_symbol_reconcile(
|
|
*,
|
|
portfolio_name: str,
|
|
targets: pd.DataFrame,
|
|
our_fills: pd.DataFrame,
|
|
our_positions: pd.DataFrame,
|
|
our_pnl: pd.DataFrame,
|
|
jq_fills: pd.DataFrame,
|
|
jq_positions: pd.DataFrame,
|
|
jq_pnl: pd.DataFrame,
|
|
share_tol: float,
|
|
price_rel_tol: float,
|
|
value_tol: float,
|
|
pnl_tol: float,
|
|
) -> pd.DataFrame:
|
|
target_agg = _aggregate_targets(targets, portfolio_name)
|
|
our_fill_agg = _aggregate_our_fills(our_fills, portfolio_name)
|
|
our_pos_agg = _aggregate_our_positions(our_positions, portfolio_name)
|
|
jq_fill_agg = _aggregate_jq_fills(jq_fills, portfolio_name)
|
|
jq_pos_agg = _aggregate_jq_positions(jq_positions, portfolio_name)
|
|
|
|
key_cols = ["date", "portfolio_name", "symbol_id"]
|
|
keys = []
|
|
for frame in [target_agg, our_fill_agg, our_pos_agg, jq_fill_agg, jq_pos_agg]:
|
|
if not frame.empty:
|
|
keys.append(frame[key_cols])
|
|
if not keys:
|
|
return pd.DataFrame(columns=RECONCILE_COLUMNS)
|
|
|
|
base = pd.concat(keys, ignore_index=True).drop_duplicates()
|
|
result = base.merge(target_agg, on=key_cols, how="left")
|
|
result = result.merge(our_fill_agg, on=key_cols, how="left")
|
|
result = result.merge(our_pos_agg, on=key_cols, how="left", suffixes=("", "_ourpos"))
|
|
result = result.merge(jq_fill_agg, on=key_cols, how="left")
|
|
result = result.merge(jq_pos_agg, on=key_cols, how="left", suffixes=("", "_jqpos"))
|
|
|
|
jq_symbol_cols = [col for col in result.columns if col.startswith("jq_symbol")]
|
|
jq_symbol_values = result[jq_symbol_cols].copy() if jq_symbol_cols else pd.DataFrame(index=result.index)
|
|
result["jq_symbol"] = ""
|
|
for col in jq_symbol_values.columns:
|
|
values = jq_symbol_values[col].fillna("").astype(str)
|
|
result["jq_symbol"] = result["jq_symbol"].mask(
|
|
result["jq_symbol"].eq("") & values.ne(""),
|
|
values,
|
|
)
|
|
result["jq_symbol"] = result.apply(
|
|
lambda row: row["jq_symbol"] or to_joinquant_symbol(row["symbol_id"]),
|
|
axis=1,
|
|
)
|
|
|
|
result["_our_present"] = (
|
|
result[["our_filled_shares", "our_position_shares", "our_position_fallback"]]
|
|
.notna()
|
|
.any(axis=1)
|
|
)
|
|
result["_jq_present"] = (
|
|
result[["jq_filled_shares", "jq_position_shares"]].notna().any(axis=1)
|
|
)
|
|
|
|
target_shares = pd.to_numeric(result["target_shares"], errors="coerce")
|
|
our_target = pd.to_numeric(result["our_target_shares"], errors="coerce")
|
|
jq_target = pd.to_numeric(result["jq_requested_shares"], errors="coerce")
|
|
result["target_shares"] = target_shares.where(target_shares.notna(), our_target)
|
|
result["target_shares"] = result["target_shares"].where(
|
|
result["target_shares"].notna(),
|
|
jq_target,
|
|
)
|
|
|
|
our_position = pd.to_numeric(result["our_position_shares"], errors="coerce")
|
|
our_position_fallback = pd.to_numeric(result["our_position_fallback"], errors="coerce")
|
|
result["our_position_shares"] = our_position.where(
|
|
our_position.notna(),
|
|
our_position_fallback,
|
|
)
|
|
|
|
for col in [
|
|
"target_shares", "our_filled_shares", "jq_filled_shares",
|
|
"our_position_shares", "jq_position_shares", "our_cost", "jq_cost",
|
|
]:
|
|
result[col] = pd.to_numeric(result[col], errors="coerce").fillna(0.0)
|
|
|
|
result["filled_share_diff"] = result["our_filled_shares"] - result["jq_filled_shares"]
|
|
result["position_share_diff"] = result["our_position_shares"] - result["jq_position_shares"]
|
|
result["trade_price_diff"] = np.where(
|
|
result["our_trade_price"].notna() & result["jq_trade_price"].notna(),
|
|
result["our_trade_price"] - result["jq_trade_price"],
|
|
np.nan,
|
|
)
|
|
result["cost_diff"] = result["our_cost"] - result["jq_cost"]
|
|
|
|
our_daily = _portfolio_frame(our_pnl, portfolio_name, "our")
|
|
jq_daily = _portfolio_frame(jq_pnl, portfolio_name, "jq")
|
|
pnl_daily = our_daily.merge(jq_daily, on=["date", "portfolio_name"], how="outer")
|
|
if not pnl_daily.empty:
|
|
pnl_daily["our_pnl"] = pd.to_numeric(pnl_daily.get("our_pnl"), errors="coerce").fillna(0.0)
|
|
pnl_daily["jq_pnl"] = pd.to_numeric(pnl_daily.get("jq_pnl"), errors="coerce").fillna(0.0)
|
|
pnl_daily["pnl_diff"] = pnl_daily["our_pnl"] - pnl_daily["jq_pnl"]
|
|
result = result.merge(
|
|
pnl_daily[["date", "portfolio_name", "our_pnl", "jq_pnl", "pnl_diff"]],
|
|
on=["date", "portfolio_name"],
|
|
how="left",
|
|
)
|
|
else:
|
|
result["our_pnl"] = 0.0
|
|
result["jq_pnl"] = 0.0
|
|
result["pnl_diff"] = 0.0
|
|
for col in ["our_pnl", "jq_pnl", "pnl_diff"]:
|
|
result[col] = pd.to_numeric(result[col], errors="coerce").fillna(0.0)
|
|
|
|
result["raw_status"] = result.get("raw_status", "").fillna("")
|
|
result["diff_reason"] = result.apply(
|
|
_classify_symbol_row,
|
|
axis=1,
|
|
share_tol=share_tol,
|
|
price_rel_tol=price_rel_tol,
|
|
value_tol=value_tol,
|
|
pnl_tol=pnl_tol,
|
|
)
|
|
|
|
return result[RECONCILE_COLUMNS].sort_values(
|
|
["date", "portfolio_name", "symbol_id"]
|
|
).reset_index(drop=True)
|
|
|
|
|
|
def _build_portfolio_summary(
|
|
*,
|
|
portfolio_name: str,
|
|
our_pnl: pd.DataFrame,
|
|
jq_pnl: pd.DataFrame,
|
|
value_tol: float,
|
|
pnl_tol: float,
|
|
) -> pd.DataFrame:
|
|
our = _portfolio_frame(our_pnl, portfolio_name, "our")
|
|
jq = _portfolio_frame(jq_pnl, portfolio_name, "jq")
|
|
if our.empty and jq.empty:
|
|
return pd.DataFrame(columns=[
|
|
"date", "portfolio_name", "diff_reason",
|
|
"our_pnl", "jq_pnl", "pnl_diff",
|
|
])
|
|
summary = our.merge(jq, on=["date", "portfolio_name"], how="outer")
|
|
summary["_our_present"] = summary.filter(regex=r"^our_").notna().any(axis=1)
|
|
summary["_jq_present"] = summary.filter(regex=r"^jq_").notna().any(axis=1)
|
|
metrics = ["gross_exposure", "net_exposure", "cash", "total_value", "pnl", "cost", "turnover"]
|
|
for metric in metrics:
|
|
our_col = f"our_{metric}"
|
|
jq_col = f"jq_{metric}"
|
|
if our_col not in summary.columns:
|
|
summary[our_col] = np.nan
|
|
if jq_col not in summary.columns:
|
|
summary[jq_col] = np.nan
|
|
summary[f"{metric}_diff"] = (
|
|
pd.to_numeric(summary[our_col], errors="coerce").fillna(0.0)
|
|
- pd.to_numeric(summary[jq_col], errors="coerce").fillna(0.0)
|
|
)
|
|
summary["our_cumulative_pnl"] = (
|
|
pd.to_numeric(summary["our_pnl"], errors="coerce").fillna(0.0).cumsum()
|
|
)
|
|
summary["jq_cumulative_pnl"] = (
|
|
pd.to_numeric(summary["jq_pnl"], errors="coerce").fillna(0.0).cumsum()
|
|
)
|
|
summary["cumulative_pnl_diff"] = summary["our_cumulative_pnl"] - summary["jq_cumulative_pnl"]
|
|
summary["diff_reason"] = summary.apply(
|
|
_classify_portfolio_row,
|
|
axis=1,
|
|
value_tol=value_tol,
|
|
pnl_tol=pnl_tol,
|
|
)
|
|
summary = summary.drop(columns=["_our_present", "_jq_present"])
|
|
return summary.sort_values(["date", "portfolio_name"]).reset_index(drop=True)
|
|
|
|
|
|
def _write_summary_md(
|
|
path: Path,
|
|
*,
|
|
portfolio_name: str,
|
|
symbol_report: pd.DataFrame,
|
|
portfolio_summary: pd.DataFrame,
|
|
) -> None:
|
|
symbol_counts = (
|
|
symbol_report["diff_reason"].value_counts().sort_index()
|
|
if not symbol_report.empty else pd.Series(dtype=int)
|
|
)
|
|
portfolio_counts = (
|
|
portfolio_summary["diff_reason"].value_counts().sort_index()
|
|
if not portfolio_summary.empty else pd.Series(dtype=int)
|
|
)
|
|
lines = [
|
|
"# JoinQuant Reconciliation Summary",
|
|
"",
|
|
f"Portfolio: `{portfolio_name}`",
|
|
"",
|
|
"## Per-symbol Difference Counts",
|
|
"",
|
|
]
|
|
if symbol_counts.empty:
|
|
lines.append("No per-symbol rows were produced.")
|
|
else:
|
|
for reason, count in symbol_counts.items():
|
|
lines.append(f"- {reason}: {int(count)}")
|
|
lines.extend(["", "## Daily Portfolio Difference Counts", ""])
|
|
if portfolio_counts.empty:
|
|
lines.append("No daily portfolio rows were produced.")
|
|
else:
|
|
for reason, count in portfolio_counts.items():
|
|
lines.append(f"- {reason}: {int(count)}")
|
|
|
|
if not portfolio_summary.empty:
|
|
lines.extend(["", "## Daily Portfolio Preview", ""])
|
|
preview_cols = [
|
|
"date", "diff_reason", "our_pnl", "jq_pnl", "pnl_diff",
|
|
"our_cost", "jq_cost", "cost_diff",
|
|
]
|
|
preview_cols = [col for col in preview_cols if col in portfolio_summary.columns]
|
|
lines.append(",".join(preview_cols))
|
|
for row in portfolio_summary[preview_cols].head(20).itertuples(index=False):
|
|
lines.append(",".join(str(value) for value in row))
|
|
|
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def reconcile_joinquant(
|
|
*,
|
|
portfolio_name: str,
|
|
targets_dir: str | Path,
|
|
our_fills_path: str | Path,
|
|
our_positions_path: str | Path,
|
|
our_pnl_path: str | Path,
|
|
jq_fills_path: str | Path,
|
|
jq_positions_path: str | Path,
|
|
jq_pnl_path: str | Path,
|
|
out_dir: str | Path = "plugins_output/joinquant/reconcile",
|
|
share_tolerance: float = 0.0,
|
|
price_rel_tolerance: float = 1e-4,
|
|
pnl_tolerance: float = 1.0,
|
|
booksize: float | None = None,
|
|
) -> dict[str, Path]:
|
|
"""Reconcile JoinQuant output against internal simulator output."""
|
|
targets = _load_targets(targets_dir, portfolio_name)
|
|
our_fills = _read_parquet(our_fills_path)
|
|
our_positions = _read_parquet(our_positions_path)
|
|
our_pnl = _read_parquet(our_pnl_path)
|
|
jq_fills = _read_parquet(jq_fills_path)
|
|
jq_positions = _read_parquet(jq_positions_path)
|
|
jq_pnl = _read_parquet(jq_pnl_path)
|
|
|
|
inferred_booksize = booksize or _infer_booksize(targets, our_pnl, jq_pnl)
|
|
value_tol = max(1.0, 1e-6 * float(inferred_booksize))
|
|
|
|
symbol_report = _build_symbol_reconcile(
|
|
portfolio_name=portfolio_name,
|
|
targets=targets,
|
|
our_fills=our_fills,
|
|
our_positions=our_positions,
|
|
our_pnl=our_pnl,
|
|
jq_fills=jq_fills,
|
|
jq_positions=jq_positions,
|
|
jq_pnl=jq_pnl,
|
|
share_tol=share_tolerance,
|
|
price_rel_tol=price_rel_tolerance,
|
|
value_tol=value_tol,
|
|
pnl_tol=pnl_tolerance,
|
|
)
|
|
portfolio_summary = _build_portfolio_summary(
|
|
portfolio_name=portfolio_name,
|
|
our_pnl=our_pnl,
|
|
jq_pnl=jq_pnl,
|
|
value_tol=value_tol,
|
|
pnl_tol=pnl_tolerance,
|
|
)
|
|
|
|
root = Path(out_dir) / portfolio_name
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
paths = {
|
|
"daily_reconcile": root / "daily_reconcile.pq",
|
|
"summary_csv": root / "summary.csv",
|
|
"summary_md": root / "summary.md",
|
|
}
|
|
symbol_report.to_parquet(paths["daily_reconcile"], index=False)
|
|
portfolio_summary.to_csv(paths["summary_csv"], index=False)
|
|
_write_summary_md(
|
|
paths["summary_md"],
|
|
portfolio_name=portfolio_name,
|
|
symbol_report=symbol_report,
|
|
portfolio_summary=portfolio_summary,
|
|
)
|
|
return paths
|