Add JoinQuant comparison plugin
This commit is contained in:
Yuxuan Yan
@@ -0,0 +1,607 @@
|
||||
"""Reconcile internal simulator output against normalized JoinQuant output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from plugins.joinquant.schema import (
|
||||
JOINQUANT_FILL_COLUMNS,
|
||||
JOINQUANT_PNL_COLUMNS,
|
||||
JOINQUANT_POSITION_COLUMNS,
|
||||
JOINQUANT_TARGET_COLUMNS,
|
||||
RECONCILE_COLUMNS,
|
||||
)
|
||||
from plugins.joinquant.symbols import to_joinquant_symbol
|
||||
|
||||
|
||||
def _date_text(value: object) -> str:
|
||||
if pd.isna(value):
|
||||
return ""
|
||||
return pd.Timestamp(value).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _read_parquet(path: str | Path | None) -> pd.DataFrame:
|
||||
if path is None:
|
||||
return pd.DataFrame()
|
||||
return pd.read_parquet(path)
|
||||
|
||||
|
||||
def _normalize_common(df: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
||||
out = df.copy()
|
||||
if "date" in out.columns:
|
||||
out["date"] = out["date"].map(_date_text)
|
||||
else:
|
||||
out["date"] = ""
|
||||
if "portfolio_name" not in out.columns:
|
||||
out["portfolio_name"] = portfolio_name
|
||||
out["portfolio_name"] = out["portfolio_name"].fillna(portfolio_name).astype(str)
|
||||
if "symbol_id" in out.columns:
|
||||
out["symbol_id"] = out["symbol_id"].fillna("").astype(str)
|
||||
if "jq_symbol" not in out.columns and "symbol_id" in out.columns:
|
||||
out["jq_symbol"] = out["symbol_id"].map(
|
||||
lambda s: to_joinquant_symbol(s) if s else ""
|
||||
)
|
||||
elif "jq_symbol" in out.columns:
|
||||
out["jq_symbol"] = out["jq_symbol"].fillna("").astype(str)
|
||||
return out
|
||||
|
||||
|
||||
def _numeric(df: pd.DataFrame, column: str, default: float = 0.0) -> pd.Series:
|
||||
if column not in df.columns:
|
||||
return pd.Series([default] * len(df), index=df.index, dtype=float)
|
||||
return pd.to_numeric(df[column], errors="coerce").replace([np.inf, -np.inf], np.nan)
|
||||
|
||||
|
||||
def _weighted_price(group: pd.DataFrame, price_col: str, shares_col: str) -> float:
|
||||
prices = pd.to_numeric(group[price_col], errors="coerce")
|
||||
shares = pd.to_numeric(group[shares_col], errors="coerce").abs()
|
||||
valid = prices.notna() & shares.notna() & (shares > 0)
|
||||
if not valid.any():
|
||||
return np.nan
|
||||
return float(np.average(prices[valid], weights=shares[valid]))
|
||||
|
||||
|
||||
def _load_targets(targets_dir: str | Path, portfolio_name: str) -> pd.DataFrame:
|
||||
root = Path(targets_dir)
|
||||
if not root.exists():
|
||||
return pd.DataFrame(columns=JOINQUANT_TARGET_COLUMNS)
|
||||
|
||||
files_by_stem: dict[str, Path] = {}
|
||||
for path in sorted(root.glob("*.csv")):
|
||||
files_by_stem[path.stem] = path
|
||||
for path in sorted(root.glob("*.parquet")):
|
||||
files_by_stem[path.stem] = path
|
||||
|
||||
frames: list[pd.DataFrame] = []
|
||||
for path in files_by_stem.values():
|
||||
if path.suffix == ".parquet":
|
||||
frame = pd.read_parquet(path)
|
||||
else:
|
||||
frame = pd.read_csv(path)
|
||||
frames.append(frame)
|
||||
|
||||
if not frames:
|
||||
return pd.DataFrame(columns=JOINQUANT_TARGET_COLUMNS)
|
||||
targets = pd.concat(frames, ignore_index=True)
|
||||
targets = _normalize_common(targets, portfolio_name)
|
||||
targets = targets[targets["portfolio_name"].astype(str) == portfolio_name]
|
||||
if "target_shares" not in targets.columns:
|
||||
targets["target_shares"] = 0
|
||||
return targets.reindex(columns=JOINQUANT_TARGET_COLUMNS)
|
||||
|
||||
|
||||
def _aggregate_targets(targets: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
||||
if targets.empty:
|
||||
return pd.DataFrame(columns=["date", "portfolio_name", "symbol_id", "jq_symbol", "target_shares"])
|
||||
targets = _normalize_common(targets, portfolio_name)
|
||||
targets["target_shares"] = _numeric(targets, "target_shares", 0.0)
|
||||
grouped = (
|
||||
targets.groupby(["date", "portfolio_name", "symbol_id"], as_index=False)
|
||||
.agg(jq_symbol=("jq_symbol", "last"), target_shares=("target_shares", "last"))
|
||||
)
|
||||
return grouped
|
||||
|
||||
|
||||
def _aggregate_our_fills(fills: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
||||
if fills.empty:
|
||||
return pd.DataFrame(columns=[
|
||||
"date", "portfolio_name", "symbol_id", "our_filled_shares",
|
||||
"our_position_shares", "our_cost", "our_trade_price", "our_blocked",
|
||||
"our_target_shares",
|
||||
])
|
||||
fills = _normalize_common(fills, portfolio_name)
|
||||
fills["traded_shares"] = _numeric(fills, "traded_shares", 0.0)
|
||||
fills["realized_shares"] = _numeric(fills, "realized_shares", np.nan)
|
||||
fills["trade_cost"] = _numeric(fills, "trade_cost", 0.0).fillna(0.0)
|
||||
fills["target_shares"] = _numeric(fills, "target_shares", np.nan)
|
||||
fills["blocked"] = _numeric(fills, "blocked", 0.0).fillna(0.0)
|
||||
price_col = next(
|
||||
(col for col in ["trade_price", "fill_price", "execution_price", "price"] if col in fills.columns),
|
||||
None,
|
||||
)
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
for key, group in fills.groupby(["date", "portfolio_name", "symbol_id"], sort=False):
|
||||
row = {
|
||||
"date": key[0],
|
||||
"portfolio_name": key[1],
|
||||
"symbol_id": key[2],
|
||||
"our_filled_shares": float(group["traded_shares"].sum()),
|
||||
"our_position_shares": float(group["realized_shares"].dropna().iloc[-1])
|
||||
if group["realized_shares"].notna().any() else np.nan,
|
||||
"our_cost": float(group["trade_cost"].sum()),
|
||||
"our_trade_price": _weighted_price(group, price_col, "traded_shares")
|
||||
if price_col else np.nan,
|
||||
"our_blocked": int(group["blocked"].max()),
|
||||
"our_target_shares": float(group["target_shares"].dropna().iloc[-1])
|
||||
if group["target_shares"].notna().any() else np.nan,
|
||||
}
|
||||
rows.append(row)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _aggregate_our_positions(positions: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
||||
if positions.empty:
|
||||
return pd.DataFrame(columns=[
|
||||
"date", "portfolio_name", "symbol_id", "jq_symbol",
|
||||
"our_position_fallback", "our_position_price",
|
||||
])
|
||||
positions = _normalize_common(positions, portfolio_name)
|
||||
positions["position_shares"] = _numeric(positions, "position_shares", np.nan)
|
||||
positions["price"] = _numeric(positions, "price", np.nan)
|
||||
return (
|
||||
positions.groupby(["date", "portfolio_name", "symbol_id"], as_index=False)
|
||||
.agg(
|
||||
jq_symbol=("jq_symbol", "last"),
|
||||
our_position_fallback=("position_shares", "last"),
|
||||
our_position_price=("price", "last"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _aggregate_jq_fills(fills: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
||||
if fills.empty:
|
||||
return pd.DataFrame(columns=[
|
||||
"date", "portfolio_name", "symbol_id", "jq_filled_shares",
|
||||
"jq_trade_price", "jq_cost", "jq_blocked", "jq_requested_shares",
|
||||
"raw_status",
|
||||
])
|
||||
fills = _normalize_common(fills, portfolio_name)
|
||||
for col in JOINQUANT_FILL_COLUMNS:
|
||||
if col not in fills.columns:
|
||||
fills[col] = np.nan
|
||||
fills["filled_shares"] = _numeric(fills, "filled_shares", 0.0).fillna(0.0)
|
||||
fills["requested_shares"] = _numeric(fills, "requested_shares", np.nan)
|
||||
fills["fill_price"] = _numeric(fills, "fill_price", np.nan)
|
||||
fills["trade_cost"] = _numeric(fills, "trade_cost", 0.0).fillna(0.0)
|
||||
fills["blocked"] = _numeric(fills, "blocked", 0.0).fillna(0.0)
|
||||
fills["raw_status"] = fills["raw_status"].fillna("").astype(str)
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
for key, group in fills.groupby(["date", "portfolio_name", "symbol_id"], sort=False):
|
||||
row = {
|
||||
"date": key[0],
|
||||
"portfolio_name": key[1],
|
||||
"symbol_id": key[2],
|
||||
"jq_filled_shares": float(group["filled_shares"].sum()),
|
||||
"jq_trade_price": _weighted_price(group, "fill_price", "filled_shares"),
|
||||
"jq_cost": float(group["trade_cost"].sum()),
|
||||
"jq_blocked": int(group["blocked"].max()),
|
||||
"jq_requested_shares": float(group["requested_shares"].dropna().iloc[-1])
|
||||
if group["requested_shares"].notna().any() else np.nan,
|
||||
"raw_status": ";".join([s for s in group["raw_status"].astype(str) if s]),
|
||||
}
|
||||
rows.append(row)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def _aggregate_jq_positions(positions: pd.DataFrame, portfolio_name: str) -> pd.DataFrame:
|
||||
if positions.empty:
|
||||
return pd.DataFrame(columns=[
|
||||
"date", "portfolio_name", "symbol_id", "jq_symbol", "jq_position_shares",
|
||||
])
|
||||
positions = _normalize_common(positions, portfolio_name)
|
||||
for col in JOINQUANT_POSITION_COLUMNS:
|
||||
if col not in positions.columns:
|
||||
positions[col] = np.nan
|
||||
positions["position_shares"] = _numeric(positions, "position_shares", np.nan)
|
||||
return (
|
||||
positions.groupby(["date", "portfolio_name", "symbol_id"], as_index=False)
|
||||
.agg(jq_symbol=("jq_symbol", "last"), jq_position_shares=("position_shares", "last"))
|
||||
)
|
||||
|
||||
|
||||
def _portfolio_frame(df: pd.DataFrame, portfolio_name: str, prefix: str) -> pd.DataFrame:
|
||||
if df.empty:
|
||||
return pd.DataFrame(columns=["date", "portfolio_name"])
|
||||
out = _normalize_common(df, portfolio_name)
|
||||
for col in JOINQUANT_PNL_COLUMNS:
|
||||
if col not in out.columns:
|
||||
out[col] = np.nan
|
||||
keep = ["date", "portfolio_name", "gross_exposure", "net_exposure", "cash", "total_value", "pnl", "cost", "turnover"]
|
||||
out = out[keep].copy()
|
||||
for col in keep[2:]:
|
||||
out[col] = pd.to_numeric(out[col], errors="coerce")
|
||||
out = out.groupby(["date", "portfolio_name"], as_index=False).last()
|
||||
return out.rename(columns={col: f"{prefix}_{col}" for col in keep[2:]})
|
||||
|
||||
|
||||
def _infer_booksize(targets: pd.DataFrame, our_pnl: pd.DataFrame, jq_pnl: pd.DataFrame) -> float:
|
||||
candidates: list[float] = []
|
||||
if "target_value" in targets.columns:
|
||||
gross = (
|
||||
pd.to_numeric(targets["target_value"], errors="coerce")
|
||||
.abs()
|
||||
.replace([np.inf, -np.inf], np.nan)
|
||||
.dropna()
|
||||
.sum()
|
||||
)
|
||||
if gross > 0:
|
||||
candidates.append(float(gross))
|
||||
for df in (our_pnl, jq_pnl):
|
||||
if "gross_exposure" in df.columns:
|
||||
val = pd.to_numeric(df["gross_exposure"], errors="coerce").max()
|
||||
if pd.notna(val) and val > 0:
|
||||
candidates.append(float(val))
|
||||
return max(candidates) if candidates else 1.0
|
||||
|
||||
|
||||
def _status_reason(raw_status: object) -> str | None:
|
||||
text = str(raw_status or "").lower()
|
||||
if "suspend" in text or "halt" in text:
|
||||
return "SUSPENSION"
|
||||
if "limit_up" in text or "limit up" in text or "up_limit" in text:
|
||||
return "LIMIT_UP_BLOCK"
|
||||
if "limit_down" in text or "limit down" in text or "down_limit" in text:
|
||||
return "LIMIT_DOWN_BLOCK"
|
||||
if "volume" in text or "liquid" in text:
|
||||
return "VOLUME_OR_LIQUIDITY"
|
||||
if "cash" in text or "margin" in text:
|
||||
return "CASH_CONSTRAINT"
|
||||
return None
|
||||
|
||||
|
||||
def _classify_symbol_row(
|
||||
row: pd.Series,
|
||||
*,
|
||||
share_tol: float,
|
||||
price_rel_tol: float,
|
||||
value_tol: float,
|
||||
pnl_tol: float,
|
||||
) -> str:
|
||||
target = row.get("target_shares", np.nan)
|
||||
filled_diff = abs(row.get("filled_share_diff", 0.0))
|
||||
position_diff = abs(row.get("position_share_diff", 0.0))
|
||||
our_present = bool(row.get("_our_present", False))
|
||||
jq_present = bool(row.get("_jq_present", False))
|
||||
|
||||
if pd.notna(target) and target < 0 and (filled_diff > share_tol or position_diff > share_tol or not jq_present):
|
||||
return "SHORT_NOT_SUPPORTED"
|
||||
if not jq_present and (our_present or pd.notna(target)):
|
||||
return "MISSING_IN_JOINQUANT"
|
||||
if not our_present and jq_present:
|
||||
return "MISSING_IN_OUR_SYSTEM"
|
||||
|
||||
status_reason = _status_reason(row.get("raw_status", ""))
|
||||
if (filled_diff > share_tol or position_diff > share_tol) and status_reason:
|
||||
return status_reason
|
||||
if filled_diff > share_tol or position_diff > share_tol:
|
||||
return "UNKNOWN"
|
||||
|
||||
our_price = row.get("our_trade_price", np.nan)
|
||||
jq_price = row.get("jq_trade_price", np.nan)
|
||||
if pd.notna(our_price) and pd.notna(jq_price):
|
||||
denom = max(abs(float(our_price)), abs(float(jq_price)), 1.0)
|
||||
if abs(float(our_price) - float(jq_price)) > price_rel_tol * denom:
|
||||
return "PRICE_MISMATCH"
|
||||
|
||||
if abs(row.get("cost_diff", 0.0)) > value_tol:
|
||||
return "COST_MODEL"
|
||||
if abs(row.get("pnl_diff", 0.0)) > pnl_tol:
|
||||
return "UNKNOWN"
|
||||
return "MATCH"
|
||||
|
||||
|
||||
def _classify_portfolio_row(row: pd.Series, value_tol: float, pnl_tol: float) -> str:
|
||||
our_present = bool(row.get("_our_present", False))
|
||||
jq_present = bool(row.get("_jq_present", False))
|
||||
if not jq_present and our_present:
|
||||
return "MISSING_IN_JOINQUANT"
|
||||
if not our_present and jq_present:
|
||||
return "MISSING_IN_OUR_SYSTEM"
|
||||
if abs(row.get("cost_diff", 0.0)) > value_tol:
|
||||
return "COST_MODEL"
|
||||
if abs(row.get("pnl_diff", 0.0)) > pnl_tol:
|
||||
return "UNKNOWN"
|
||||
return "MATCH"
|
||||
|
||||
|
||||
def _build_symbol_reconcile(
|
||||
*,
|
||||
portfolio_name: str,
|
||||
targets: pd.DataFrame,
|
||||
our_fills: pd.DataFrame,
|
||||
our_positions: pd.DataFrame,
|
||||
our_pnl: pd.DataFrame,
|
||||
jq_fills: pd.DataFrame,
|
||||
jq_positions: pd.DataFrame,
|
||||
jq_pnl: pd.DataFrame,
|
||||
share_tol: float,
|
||||
price_rel_tol: float,
|
||||
value_tol: float,
|
||||
pnl_tol: float,
|
||||
) -> pd.DataFrame:
|
||||
target_agg = _aggregate_targets(targets, portfolio_name)
|
||||
our_fill_agg = _aggregate_our_fills(our_fills, portfolio_name)
|
||||
our_pos_agg = _aggregate_our_positions(our_positions, portfolio_name)
|
||||
jq_fill_agg = _aggregate_jq_fills(jq_fills, portfolio_name)
|
||||
jq_pos_agg = _aggregate_jq_positions(jq_positions, portfolio_name)
|
||||
|
||||
key_cols = ["date", "portfolio_name", "symbol_id"]
|
||||
keys = []
|
||||
for frame in [target_agg, our_fill_agg, our_pos_agg, jq_fill_agg, jq_pos_agg]:
|
||||
if not frame.empty:
|
||||
keys.append(frame[key_cols])
|
||||
if not keys:
|
||||
return pd.DataFrame(columns=RECONCILE_COLUMNS)
|
||||
|
||||
base = pd.concat(keys, ignore_index=True).drop_duplicates()
|
||||
result = base.merge(target_agg, on=key_cols, how="left")
|
||||
result = result.merge(our_fill_agg, on=key_cols, how="left")
|
||||
result = result.merge(our_pos_agg, on=key_cols, how="left", suffixes=("", "_ourpos"))
|
||||
result = result.merge(jq_fill_agg, on=key_cols, how="left")
|
||||
result = result.merge(jq_pos_agg, on=key_cols, how="left", suffixes=("", "_jqpos"))
|
||||
|
||||
jq_symbol_cols = [col for col in result.columns if col.startswith("jq_symbol")]
|
||||
jq_symbol_values = result[jq_symbol_cols].copy() if jq_symbol_cols else pd.DataFrame(index=result.index)
|
||||
result["jq_symbol"] = ""
|
||||
for col in jq_symbol_values.columns:
|
||||
values = jq_symbol_values[col].fillna("").astype(str)
|
||||
result["jq_symbol"] = result["jq_symbol"].mask(
|
||||
result["jq_symbol"].eq("") & values.ne(""),
|
||||
values,
|
||||
)
|
||||
result["jq_symbol"] = result.apply(
|
||||
lambda row: row["jq_symbol"] or to_joinquant_symbol(row["symbol_id"]),
|
||||
axis=1,
|
||||
)
|
||||
|
||||
result["_our_present"] = (
|
||||
result[["our_filled_shares", "our_position_shares", "our_position_fallback"]]
|
||||
.notna()
|
||||
.any(axis=1)
|
||||
)
|
||||
result["_jq_present"] = (
|
||||
result[["jq_filled_shares", "jq_position_shares"]].notna().any(axis=1)
|
||||
)
|
||||
|
||||
target_shares = pd.to_numeric(result["target_shares"], errors="coerce")
|
||||
our_target = pd.to_numeric(result["our_target_shares"], errors="coerce")
|
||||
jq_target = pd.to_numeric(result["jq_requested_shares"], errors="coerce")
|
||||
result["target_shares"] = target_shares.where(target_shares.notna(), our_target)
|
||||
result["target_shares"] = result["target_shares"].where(
|
||||
result["target_shares"].notna(),
|
||||
jq_target,
|
||||
)
|
||||
|
||||
our_position = pd.to_numeric(result["our_position_shares"], errors="coerce")
|
||||
our_position_fallback = pd.to_numeric(result["our_position_fallback"], errors="coerce")
|
||||
result["our_position_shares"] = our_position.where(
|
||||
our_position.notna(),
|
||||
our_position_fallback,
|
||||
)
|
||||
|
||||
for col in [
|
||||
"target_shares", "our_filled_shares", "jq_filled_shares",
|
||||
"our_position_shares", "jq_position_shares", "our_cost", "jq_cost",
|
||||
]:
|
||||
result[col] = pd.to_numeric(result[col], errors="coerce").fillna(0.0)
|
||||
|
||||
result["filled_share_diff"] = result["our_filled_shares"] - result["jq_filled_shares"]
|
||||
result["position_share_diff"] = result["our_position_shares"] - result["jq_position_shares"]
|
||||
result["trade_price_diff"] = np.where(
|
||||
result["our_trade_price"].notna() & result["jq_trade_price"].notna(),
|
||||
result["our_trade_price"] - result["jq_trade_price"],
|
||||
np.nan,
|
||||
)
|
||||
result["cost_diff"] = result["our_cost"] - result["jq_cost"]
|
||||
|
||||
our_daily = _portfolio_frame(our_pnl, portfolio_name, "our")
|
||||
jq_daily = _portfolio_frame(jq_pnl, portfolio_name, "jq")
|
||||
pnl_daily = our_daily.merge(jq_daily, on=["date", "portfolio_name"], how="outer")
|
||||
if not pnl_daily.empty:
|
||||
pnl_daily["our_pnl"] = pd.to_numeric(pnl_daily.get("our_pnl"), errors="coerce").fillna(0.0)
|
||||
pnl_daily["jq_pnl"] = pd.to_numeric(pnl_daily.get("jq_pnl"), errors="coerce").fillna(0.0)
|
||||
pnl_daily["pnl_diff"] = pnl_daily["our_pnl"] - pnl_daily["jq_pnl"]
|
||||
result = result.merge(
|
||||
pnl_daily[["date", "portfolio_name", "our_pnl", "jq_pnl", "pnl_diff"]],
|
||||
on=["date", "portfolio_name"],
|
||||
how="left",
|
||||
)
|
||||
else:
|
||||
result["our_pnl"] = 0.0
|
||||
result["jq_pnl"] = 0.0
|
||||
result["pnl_diff"] = 0.0
|
||||
for col in ["our_pnl", "jq_pnl", "pnl_diff"]:
|
||||
result[col] = pd.to_numeric(result[col], errors="coerce").fillna(0.0)
|
||||
|
||||
result["raw_status"] = result.get("raw_status", "").fillna("")
|
||||
result["diff_reason"] = result.apply(
|
||||
_classify_symbol_row,
|
||||
axis=1,
|
||||
share_tol=share_tol,
|
||||
price_rel_tol=price_rel_tol,
|
||||
value_tol=value_tol,
|
||||
pnl_tol=pnl_tol,
|
||||
)
|
||||
|
||||
return result[RECONCILE_COLUMNS].sort_values(
|
||||
["date", "portfolio_name", "symbol_id"]
|
||||
).reset_index(drop=True)
|
||||
|
||||
|
||||
def _build_portfolio_summary(
|
||||
*,
|
||||
portfolio_name: str,
|
||||
our_pnl: pd.DataFrame,
|
||||
jq_pnl: pd.DataFrame,
|
||||
value_tol: float,
|
||||
pnl_tol: float,
|
||||
) -> pd.DataFrame:
|
||||
our = _portfolio_frame(our_pnl, portfolio_name, "our")
|
||||
jq = _portfolio_frame(jq_pnl, portfolio_name, "jq")
|
||||
if our.empty and jq.empty:
|
||||
return pd.DataFrame(columns=[
|
||||
"date", "portfolio_name", "diff_reason",
|
||||
"our_pnl", "jq_pnl", "pnl_diff",
|
||||
])
|
||||
summary = our.merge(jq, on=["date", "portfolio_name"], how="outer")
|
||||
summary["_our_present"] = summary.filter(regex=r"^our_").notna().any(axis=1)
|
||||
summary["_jq_present"] = summary.filter(regex=r"^jq_").notna().any(axis=1)
|
||||
metrics = ["gross_exposure", "net_exposure", "cash", "total_value", "pnl", "cost", "turnover"]
|
||||
for metric in metrics:
|
||||
our_col = f"our_{metric}"
|
||||
jq_col = f"jq_{metric}"
|
||||
if our_col not in summary.columns:
|
||||
summary[our_col] = np.nan
|
||||
if jq_col not in summary.columns:
|
||||
summary[jq_col] = np.nan
|
||||
summary[f"{metric}_diff"] = (
|
||||
pd.to_numeric(summary[our_col], errors="coerce").fillna(0.0)
|
||||
- pd.to_numeric(summary[jq_col], errors="coerce").fillna(0.0)
|
||||
)
|
||||
summary["our_cumulative_pnl"] = (
|
||||
pd.to_numeric(summary["our_pnl"], errors="coerce").fillna(0.0).cumsum()
|
||||
)
|
||||
summary["jq_cumulative_pnl"] = (
|
||||
pd.to_numeric(summary["jq_pnl"], errors="coerce").fillna(0.0).cumsum()
|
||||
)
|
||||
summary["cumulative_pnl_diff"] = summary["our_cumulative_pnl"] - summary["jq_cumulative_pnl"]
|
||||
summary["diff_reason"] = summary.apply(
|
||||
_classify_portfolio_row,
|
||||
axis=1,
|
||||
value_tol=value_tol,
|
||||
pnl_tol=pnl_tol,
|
||||
)
|
||||
summary = summary.drop(columns=["_our_present", "_jq_present"])
|
||||
return summary.sort_values(["date", "portfolio_name"]).reset_index(drop=True)
|
||||
|
||||
|
||||
def _write_summary_md(
|
||||
path: Path,
|
||||
*,
|
||||
portfolio_name: str,
|
||||
symbol_report: pd.DataFrame,
|
||||
portfolio_summary: pd.DataFrame,
|
||||
) -> None:
|
||||
symbol_counts = (
|
||||
symbol_report["diff_reason"].value_counts().sort_index()
|
||||
if not symbol_report.empty else pd.Series(dtype=int)
|
||||
)
|
||||
portfolio_counts = (
|
||||
portfolio_summary["diff_reason"].value_counts().sort_index()
|
||||
if not portfolio_summary.empty else pd.Series(dtype=int)
|
||||
)
|
||||
lines = [
|
||||
"# JoinQuant Reconciliation Summary",
|
||||
"",
|
||||
f"Portfolio: `{portfolio_name}`",
|
||||
"",
|
||||
"## Per-symbol Difference Counts",
|
||||
"",
|
||||
]
|
||||
if symbol_counts.empty:
|
||||
lines.append("No per-symbol rows were produced.")
|
||||
else:
|
||||
for reason, count in symbol_counts.items():
|
||||
lines.append(f"- {reason}: {int(count)}")
|
||||
lines.extend(["", "## Daily Portfolio Difference Counts", ""])
|
||||
if portfolio_counts.empty:
|
||||
lines.append("No daily portfolio rows were produced.")
|
||||
else:
|
||||
for reason, count in portfolio_counts.items():
|
||||
lines.append(f"- {reason}: {int(count)}")
|
||||
|
||||
if not portfolio_summary.empty:
|
||||
lines.extend(["", "## Daily Portfolio Preview", ""])
|
||||
preview_cols = [
|
||||
"date", "diff_reason", "our_pnl", "jq_pnl", "pnl_diff",
|
||||
"our_cost", "jq_cost", "cost_diff",
|
||||
]
|
||||
preview_cols = [col for col in preview_cols if col in portfolio_summary.columns]
|
||||
lines.append(",".join(preview_cols))
|
||||
for row in portfolio_summary[preview_cols].head(20).itertuples(index=False):
|
||||
lines.append(",".join(str(value) for value in row))
|
||||
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def reconcile_joinquant(
|
||||
*,
|
||||
portfolio_name: str,
|
||||
targets_dir: str | Path,
|
||||
our_fills_path: str | Path,
|
||||
our_positions_path: str | Path,
|
||||
our_pnl_path: str | Path,
|
||||
jq_fills_path: str | Path,
|
||||
jq_positions_path: str | Path,
|
||||
jq_pnl_path: str | Path,
|
||||
out_dir: str | Path = "plugins_output/joinquant/reconcile",
|
||||
share_tolerance: float = 0.0,
|
||||
price_rel_tolerance: float = 1e-4,
|
||||
pnl_tolerance: float = 1.0,
|
||||
booksize: float | None = None,
|
||||
) -> dict[str, Path]:
|
||||
"""Reconcile JoinQuant output against internal simulator output."""
|
||||
targets = _load_targets(targets_dir, portfolio_name)
|
||||
our_fills = _read_parquet(our_fills_path)
|
||||
our_positions = _read_parquet(our_positions_path)
|
||||
our_pnl = _read_parquet(our_pnl_path)
|
||||
jq_fills = _read_parquet(jq_fills_path)
|
||||
jq_positions = _read_parquet(jq_positions_path)
|
||||
jq_pnl = _read_parquet(jq_pnl_path)
|
||||
|
||||
inferred_booksize = booksize or _infer_booksize(targets, our_pnl, jq_pnl)
|
||||
value_tol = max(1.0, 1e-6 * float(inferred_booksize))
|
||||
|
||||
symbol_report = _build_symbol_reconcile(
|
||||
portfolio_name=portfolio_name,
|
||||
targets=targets,
|
||||
our_fills=our_fills,
|
||||
our_positions=our_positions,
|
||||
our_pnl=our_pnl,
|
||||
jq_fills=jq_fills,
|
||||
jq_positions=jq_positions,
|
||||
jq_pnl=jq_pnl,
|
||||
share_tol=share_tolerance,
|
||||
price_rel_tol=price_rel_tolerance,
|
||||
value_tol=value_tol,
|
||||
pnl_tol=pnl_tolerance,
|
||||
)
|
||||
portfolio_summary = _build_portfolio_summary(
|
||||
portfolio_name=portfolio_name,
|
||||
our_pnl=our_pnl,
|
||||
jq_pnl=jq_pnl,
|
||||
value_tol=value_tol,
|
||||
pnl_tol=pnl_tolerance,
|
||||
)
|
||||
|
||||
root = Path(out_dir) / portfolio_name
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
paths = {
|
||||
"daily_reconcile": root / "daily_reconcile.pq",
|
||||
"summary_csv": root / "summary.csv",
|
||||
"summary_md": root / "summary.md",
|
||||
}
|
||||
symbol_report.to_parquet(paths["daily_reconcile"], index=False)
|
||||
portfolio_summary.to_csv(paths["summary_csv"], index=False)
|
||||
_write_summary_md(
|
||||
paths["summary_md"],
|
||||
portfolio_name=portfolio_name,
|
||||
symbol_report=symbol_report,
|
||||
portfolio_summary=portfolio_summary,
|
||||
)
|
||||
return paths
|
||||
Reference in New Issue
Block a user