Add offline workflow and coverage tests
This commit is contained in:
Yuxuan Yan
Binary file not shown.
@@ -0,0 +1,261 @@
|
||||
"""Shared deterministic test data for offline workflow tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from pipeline.common.schema import (
|
||||
ALPHA_COLUMNS,
|
||||
COMBO_COLUMNS,
|
||||
DATA_COLUMNS,
|
||||
MINUTE_BAR_COLUMNS,
|
||||
)
|
||||
|
||||
|
||||
GENERATED_SYMBOLS: tuple[str, ...] = (
|
||||
"sh600000",
|
||||
"sz000001",
|
||||
"sh600519",
|
||||
"sz300750",
|
||||
)
|
||||
|
||||
GENERATED_SYMBOL_NAMES: dict[str, str] = {
|
||||
"sh600000": "PF Bank",
|
||||
"sz000001": "Ping An Bank",
|
||||
"sh600519": "Kweichow Moutai",
|
||||
"sz300750": "CATL",
|
||||
}
|
||||
|
||||
|
||||
def generated_sessions(n_sessions: int = 12) -> pd.DatetimeIndex:
|
||||
"""Return a fixed business-day calendar used by generated fixtures."""
|
||||
return pd.bdate_range("2024-01-02", periods=n_sessions)
|
||||
|
||||
|
||||
def make_generated_daily_bars(
|
||||
n_sessions: int = 12,
|
||||
include_missing: bool = True,
|
||||
) -> pd.DataFrame:
|
||||
"""Build daily bars with explicit edge cases and no randomness.
|
||||
|
||||
The panel covers four A-share symbols and includes a suspended row, an ST
|
||||
flag, a zero-volume row, a missing symbol-date, and limit-style open/close
|
||||
moves. Values are deterministic so tests can assert exact identities.
|
||||
"""
|
||||
dates = generated_sessions(n_sessions)
|
||||
base_close = {
|
||||
"sh600000": 10.00,
|
||||
"sz000001": 15.00,
|
||||
"sh600519": 1200.00,
|
||||
"sz300750": 180.00,
|
||||
}
|
||||
returns = {
|
||||
"sh600000": [0.000, 0.012, -0.006, 0.018, 0.100, -0.014, 0.006, 0.000, 0.008, -0.011, 0.004, 0.009],
|
||||
"sz000001": [0.000, -0.008, 0.011, -0.004, 0.006, 0.000, -0.012, 0.009, 0.005, -0.007, 0.010, -0.003],
|
||||
"sh600519": [0.000, 0.006, 0.004, -0.010, 0.012, -0.006, 0.005, 0.003, -0.009, 0.007, -0.004, 0.006],
|
||||
"sz300750": [0.000, -0.010, 0.014, 0.006, -0.008, 0.011, -0.004, 0.009, -0.200, 0.012, -0.006, 0.008],
|
||||
}
|
||||
base_volume = {
|
||||
"sh600000": 1_200_000.0,
|
||||
"sz000001": 900_000.0,
|
||||
"sh600519": 80_000.0,
|
||||
"sz300750": 240_000.0,
|
||||
}
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
for sym in GENERATED_SYMBOLS:
|
||||
closes = [base_close[sym]]
|
||||
pattern = returns[sym]
|
||||
for step in range(1, n_sessions):
|
||||
ret = pattern[step % len(pattern)]
|
||||
closes.append(closes[-1] * (1.0 + ret))
|
||||
closes_arr = np.asarray(closes, dtype=float)
|
||||
precloses = np.concatenate([[closes_arr[0]], closes_arr[:-1]])
|
||||
|
||||
for i, date in enumerate(dates):
|
||||
preclose = float(precloses[i])
|
||||
close = float(closes_arr[i])
|
||||
open_price = preclose * (1.0 + 0.25 * (close / preclose - 1.0))
|
||||
high = max(open_price, close) * 1.01
|
||||
low = min(open_price, close) * 0.99
|
||||
volume = base_volume[sym] + 10_000.0 * i
|
||||
tradestatus = 1
|
||||
is_st = 0
|
||||
|
||||
if sym == "sh600000" and i == 4:
|
||||
open_price = preclose * 1.10
|
||||
close = open_price
|
||||
high = open_price
|
||||
low = open_price
|
||||
if sym == "sh600000" and i == 7:
|
||||
volume = 0.0
|
||||
if sym == "sz000001" and i == 5:
|
||||
open_price = preclose
|
||||
close = preclose
|
||||
high = preclose
|
||||
low = preclose
|
||||
volume = 0.0
|
||||
tradestatus = 0
|
||||
if sym == "sh600519" and i == 7:
|
||||
is_st = 1
|
||||
if sym == "sz300750" and i == 8:
|
||||
open_price = preclose * 0.80
|
||||
close = open_price
|
||||
high = open_price
|
||||
low = open_price
|
||||
|
||||
amount = volume * ((open_price + close) / 2.0)
|
||||
vwap = amount / volume if volume > 0 else np.nan
|
||||
pct_chg = (close / preclose - 1.0) * 100.0 if preclose else 0.0
|
||||
rows.append({
|
||||
"symbol_id": sym,
|
||||
"symbol_name": GENERATED_SYMBOL_NAMES[sym],
|
||||
"date": date,
|
||||
"open": open_price,
|
||||
"high": high,
|
||||
"low": low,
|
||||
"close": close,
|
||||
"preclose": preclose,
|
||||
"volume": volume,
|
||||
"amount": amount,
|
||||
"vwap": vwap,
|
||||
"turn": volume / 1_000_000.0,
|
||||
"pctChg": pct_chg,
|
||||
"tradestatus": tradestatus,
|
||||
"isST": is_st,
|
||||
"peTTM": 8.0 + i,
|
||||
"pbMRQ": 1.0 + 0.05 * i,
|
||||
"psTTM": 2.0 + 0.03 * i,
|
||||
"pcfNcfTTM": 5.0 + 0.1 * i,
|
||||
})
|
||||
|
||||
result = pd.DataFrame(rows)
|
||||
if include_missing and n_sessions > 6:
|
||||
missing_mask = (
|
||||
(result["symbol_id"] == "sz300750")
|
||||
& (result["date"] == dates[6])
|
||||
)
|
||||
result = result.loc[~missing_mask].copy()
|
||||
|
||||
result = result[DATA_COLUMNS]
|
||||
return result.sort_values(["date", "symbol_id"]).reset_index(drop=True)
|
||||
|
||||
|
||||
def make_generated_minute_bars(
|
||||
daily: pd.DataFrame | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""Expand generated daily bars into a tiny deterministic intraday panel."""
|
||||
daily = make_generated_daily_bars() if daily is None else daily.copy()
|
||||
rows: list[dict[str, object]] = []
|
||||
bar_times = ("09:35:00", "10:30:00", "14:55:00")
|
||||
|
||||
for daily_row in daily.sort_values(["date", "symbol_id"]).itertuples(index=False):
|
||||
if int(getattr(daily_row, "tradestatus", 1)) == 0:
|
||||
continue
|
||||
|
||||
volume = float(daily_row.volume)
|
||||
volume_slices = [0.25 * volume, 0.35 * volume, 0.40 * volume]
|
||||
prices = np.linspace(float(daily_row.open), float(daily_row.close), len(bar_times))
|
||||
for j, time_text in enumerate(bar_times):
|
||||
dt = pd.Timestamp(daily_row.date) + pd.Timedelta(time_text)
|
||||
open_price = prices[j - 1] if j else float(daily_row.open)
|
||||
close_price = float(prices[j])
|
||||
high = max(open_price, close_price) * 1.002
|
||||
low = min(open_price, close_price) * 0.998
|
||||
minute_volume = float(volume_slices[j])
|
||||
amount = minute_volume * ((open_price + close_price) / 2.0)
|
||||
rows.append({
|
||||
"symbol_id": daily_row.symbol_id,
|
||||
"symbol_name": daily_row.symbol_name,
|
||||
"datetime": dt,
|
||||
"date": pd.Timestamp(daily_row.date).normalize(),
|
||||
"time": time_text,
|
||||
"frequency": "5m",
|
||||
"open": open_price,
|
||||
"high": high,
|
||||
"low": low,
|
||||
"close": close_price,
|
||||
"volume": minute_volume,
|
||||
"amount": amount,
|
||||
"vwap": amount / minute_volume if minute_volume > 0 else np.nan,
|
||||
"adjustflag": "3",
|
||||
})
|
||||
|
||||
return pd.DataFrame(rows, columns=MINUTE_BAR_COLUMNS)
|
||||
|
||||
|
||||
def make_generated_derived_features(
|
||||
daily: pd.DataFrame | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""Return numeric daily derived values, including NaN and infinity cells."""
|
||||
daily = make_generated_daily_bars() if daily is None else daily.copy()
|
||||
keys = (
|
||||
daily[["symbol_id", "date"]]
|
||||
.drop_duplicates()
|
||||
.sort_values(["date", "symbol_id"])
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
date_rank = keys["date"].rank(method="dense").astype(float)
|
||||
symbol_rank = keys["symbol_id"].map({
|
||||
"sh600000": 1.0,
|
||||
"sz000001": 2.0,
|
||||
"sh600519": 3.0,
|
||||
"sz300750": 4.0,
|
||||
})
|
||||
out = keys.copy()
|
||||
out["toy_feature"] = symbol_rank + date_rank / 100.0
|
||||
out["finite_feature"] = symbol_rank * date_rank
|
||||
out["nan_feature"] = out["toy_feature"]
|
||||
out["inf_feature"] = out["toy_feature"]
|
||||
if len(out) >= 2:
|
||||
out.loc[out.index[0], "nan_feature"] = np.nan
|
||||
out.loc[out.index[1], "inf_feature"] = np.inf
|
||||
return out
|
||||
|
||||
|
||||
def make_generated_alpha_weights(
|
||||
alpha_name: str = "alpha_a",
|
||||
*,
|
||||
scale: float = 1.0,
|
||||
offset: float = 0.0,
|
||||
zero_date_index: int | None = None,
|
||||
n_sessions: int = 10,
|
||||
) -> pd.DataFrame:
|
||||
"""Create a deterministic long alpha grid with optional zero-gross date."""
|
||||
dates = generated_sessions(n_sessions)
|
||||
even = np.array([1.20, -0.80, 0.40, -0.80], dtype=float)
|
||||
odd = np.array([-0.60, 1.10, -0.90, 0.40], dtype=float)
|
||||
rows: list[dict[str, object]] = []
|
||||
for i, date in enumerate(dates):
|
||||
vector = even.copy() if i % 2 == 0 else odd.copy()
|
||||
vector = vector + offset
|
||||
vector = vector - vector.mean()
|
||||
if zero_date_index is not None and i == zero_date_index:
|
||||
vector = np.zeros_like(vector)
|
||||
for sym, weight in zip(GENERATED_SYMBOLS, scale * vector):
|
||||
rows.append({
|
||||
"symbol_id": sym,
|
||||
"date": date,
|
||||
"alpha_name": alpha_name,
|
||||
"weight": float(weight),
|
||||
})
|
||||
result = pd.DataFrame(rows, columns=ALPHA_COLUMNS)
|
||||
return result.sort_values(["symbol_id", "date"]).reset_index(drop=True)
|
||||
|
||||
|
||||
def make_generated_combo_weights(
|
||||
combo_name: str = "combo",
|
||||
*,
|
||||
zero_date_index: int | None = 2,
|
||||
n_sessions: int = 10,
|
||||
) -> pd.DataFrame:
|
||||
"""Create deterministic combo weights for portfolio construction tests."""
|
||||
alpha = make_generated_alpha_weights(
|
||||
"combo_source",
|
||||
zero_date_index=zero_date_index,
|
||||
n_sessions=n_sessions,
|
||||
)
|
||||
combo = alpha.rename(columns={"alpha_name": "combo_name"}).copy()
|
||||
combo["combo_name"] = combo_name
|
||||
return combo[COMBO_COLUMNS].sort_values(["symbol_id", "date"]).reset_index(drop=True)
|
||||
@@ -413,3 +413,92 @@ def test_feature_aware_alpha_reads_joined_feature_column(tmp_path):
|
||||
assert (result["alpha_name"] == "feature_run").all()
|
||||
last = result[result["date"] == result["date"].max()]
|
||||
assert last.set_index("symbol_id")["weight"].idxmax() == "sh600519"
|
||||
|
||||
|
||||
def test_feature_paths_join_multiple_files_and_normalize_dates(tmp_path):
|
||||
module_path = tmp_path / "multi_feature_alpha.py"
|
||||
module_path.write_text(textwrap.dedent('''
|
||||
import pandas as pd
|
||||
from pipeline.alpha.base import BaseAlpha
|
||||
from pipeline.alpha.registry import register_alpha
|
||||
|
||||
@register_alpha
|
||||
class MultiFeatureAlpha(BaseAlpha):
|
||||
name = "multi_feature_test_alpha"
|
||||
|
||||
def signal_from_data(
|
||||
self,
|
||||
data: pd.DataFrame,
|
||||
close: pd.DataFrame,
|
||||
) -> pd.DataFrame:
|
||||
data = data.copy()
|
||||
data["combined_feature"] = data["toy_a"] + data["toy_b"]
|
||||
signal = data.pivot_table(
|
||||
index="date",
|
||||
columns="symbol_id",
|
||||
values="combined_feature",
|
||||
aggfunc="first",
|
||||
)
|
||||
return signal.reindex(index=close.index, columns=close.columns)
|
||||
'''))
|
||||
|
||||
data = _make_data(n_days=8)
|
||||
symbol_score = {"sh600000": 1.0, "sz000001": 2.0, "sh600519": 3.0}
|
||||
|
||||
feature_a = data[["symbol_id", "date"]].copy()
|
||||
feature_a["date"] = feature_a["date"] + pd.Timedelta(hours=15)
|
||||
feature_a["toy_a"] = feature_a["symbol_id"].map(symbol_score)
|
||||
|
||||
feature_b = data[["symbol_id", "date"]].copy()
|
||||
feature_b["date"] = feature_b["date"].dt.strftime("%Y-%m-%d 09:30:00")
|
||||
feature_b["toy_b"] = feature_b["symbol_id"].map(symbol_score) * 10.0
|
||||
|
||||
feature_a_path = tmp_path / "toy_a.pq"
|
||||
feature_b_path = tmp_path / "toy_b.pq"
|
||||
feature_a.to_parquet(feature_a_path, index=False)
|
||||
feature_b.to_parquet(feature_b_path, index=False)
|
||||
|
||||
load_alpha_module(str(module_path))
|
||||
result = compute_alpha(
|
||||
data,
|
||||
"multi_feature_run",
|
||||
"multi_feature_test_alpha",
|
||||
feature_paths=[str(feature_a_path), str(feature_b_path)],
|
||||
)
|
||||
|
||||
assert list(result.columns) == ALPHA_COLUMNS
|
||||
assert (result["alpha_name"] == "multi_feature_run").all()
|
||||
last = result[result["date"] == result["date"].max()]
|
||||
assert last.set_index("symbol_id")["weight"].idxmax() == "sh600519"
|
||||
|
||||
|
||||
def test_compute_alpha_rejects_duplicate_feature_frame_columns():
|
||||
data = _make_data()
|
||||
duplicate_columns = pd.DataFrame(
|
||||
[["sh600000", pd.Timestamp("2024-01-01"), 1.0, 2.0]],
|
||||
columns=["symbol_id", "date", "toy_feature", "toy_feature"],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="duplicate columns"):
|
||||
compute_alpha(
|
||||
data,
|
||||
"bad_features",
|
||||
"reversal",
|
||||
feature_frames=[duplicate_columns],
|
||||
)
|
||||
|
||||
|
||||
def test_compute_alpha_rejects_feature_path_collision_with_daily_data(tmp_path):
|
||||
data = _make_data()
|
||||
close_collision = data[["symbol_id", "date"]].copy()
|
||||
close_collision["close"] = 1.0
|
||||
close_collision_path = tmp_path / "close_collision.pq"
|
||||
close_collision.to_parquet(close_collision_path, index=False)
|
||||
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
compute_alpha(
|
||||
data,
|
||||
"close_collision",
|
||||
"reversal",
|
||||
feature_paths=[str(close_collision_path)],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
"""CLI handoff tests for the offline daily workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from click.testing import CliRunner
|
||||
|
||||
from cli import cli
|
||||
from tests.helpers import (
|
||||
make_generated_daily_bars,
|
||||
make_generated_derived_features,
|
||||
make_generated_minute_bars,
|
||||
)
|
||||
|
||||
|
||||
FIXTURE_PATH = Path(__file__).parent / "fixtures" / "daily_bars_real_2024_01_sample.pq"
|
||||
|
||||
|
||||
def _invoke_ok(runner: CliRunner, args: list[str]):
|
||||
result = runner.invoke(cli, args)
|
||||
assert result.exit_code == 0, result.output
|
||||
return result
|
||||
|
||||
|
||||
def _invoke_error(runner: CliRunner, args: list[str]):
|
||||
result = runner.invoke(cli, args)
|
||||
assert result.exit_code != 0, result.output
|
||||
return result
|
||||
|
||||
|
||||
def test_cli_daily_workflow_handoffs_stay_in_tmp_path(tmp_path):
|
||||
runner = CliRunner()
|
||||
daily_bars = make_generated_daily_bars()
|
||||
minute_bars = make_generated_minute_bars(daily_bars)
|
||||
derived_features = make_generated_derived_features(daily_bars)
|
||||
|
||||
daily_path = tmp_path / "daily_bars.pq"
|
||||
minute_path = tmp_path / "minute_bars.pq"
|
||||
derived_input_path = tmp_path / "derived_input.pq"
|
||||
daily_bars.to_parquet(daily_path, index=False)
|
||||
minute_bars.to_parquet(minute_path, index=False)
|
||||
derived_features.to_parquet(derived_input_path, index=False)
|
||||
|
||||
ingest_dir = tmp_path / "derived_ingested"
|
||||
ingest_result = _invoke_ok(runner, [
|
||||
"derived", "ingest",
|
||||
"--input-path", str(derived_input_path),
|
||||
"--derived-name", "toy_features",
|
||||
"--output-dir", str(ingest_dir),
|
||||
])
|
||||
ingested_feature_path = ingest_dir / "toy_features.pq"
|
||||
assert "Saved derived data:" in ingest_result.output
|
||||
assert ingested_feature_path.exists()
|
||||
|
||||
validate_result = _invoke_ok(runner, [
|
||||
"derived", "validate",
|
||||
"--input-path", str(ingested_feature_path),
|
||||
])
|
||||
assert "Valid derived data:" in validate_result.output
|
||||
assert "rows" in validate_result.output
|
||||
|
||||
computed_derived_dir = tmp_path / "derived_computed"
|
||||
derived_compute_result = _invoke_ok(runner, [
|
||||
"derived", "compute",
|
||||
"--daily-path", str(daily_path),
|
||||
"--minute-path", str(minute_path),
|
||||
"--derived-type", "minute_daily_summary",
|
||||
"--derived-name", "minute_summary",
|
||||
"--output-dir", str(computed_derived_dir),
|
||||
])
|
||||
minute_summary_path = computed_derived_dir / "minute_summary.pq"
|
||||
assert "Loaded daily data:" in derived_compute_result.output
|
||||
assert "Loaded minute bars:" in derived_compute_result.output
|
||||
assert "Saved derived data:" in derived_compute_result.output
|
||||
assert minute_summary_path.exists()
|
||||
assert "minute_vwap" in pd.read_parquet(minute_summary_path).columns
|
||||
|
||||
alpha_module_path = tmp_path / "cli_feature_alpha.py"
|
||||
alpha_module_path.write_text(textwrap.dedent("""
|
||||
import pandas as pd
|
||||
from pipeline.alpha.base import BaseAlpha
|
||||
from pipeline.alpha.registry import register_alpha
|
||||
|
||||
@register_alpha
|
||||
class CliFeatureAlpha(BaseAlpha):
|
||||
name = "cli_feature_alpha_workflow"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
|
||||
def signal_from_data(
|
||||
self,
|
||||
data: pd.DataFrame,
|
||||
close: pd.DataFrame,
|
||||
) -> pd.DataFrame:
|
||||
signal = data.pivot_table(
|
||||
index="date",
|
||||
columns="symbol_id",
|
||||
values="minute_intraday_return",
|
||||
aggfunc="first",
|
||||
)
|
||||
fallback = close.pct_change(1, fill_method=None)
|
||||
feature_signal = signal.reindex(index=close.index, columns=close.columns)
|
||||
toy_signal = data.pivot_table(
|
||||
index="date",
|
||||
columns="symbol_id",
|
||||
values="toy_feature",
|
||||
aggfunc="first",
|
||||
)
|
||||
toy_signal = toy_signal.reindex(index=close.index, columns=close.columns)
|
||||
return feature_signal.fillna(fallback) + toy_signal / 1000.0
|
||||
"""))
|
||||
|
||||
alpha_dir = tmp_path / "alphas"
|
||||
alpha_result = _invoke_ok(runner, [
|
||||
"alpha", "compute",
|
||||
"--data-path", str(daily_path),
|
||||
"--feature-path", str(minute_summary_path),
|
||||
"--feature-path", str(ingested_feature_path),
|
||||
"--alpha-module", str(alpha_module_path),
|
||||
"--alpha-type", "cli_feature_alpha_workflow",
|
||||
"--alpha-name", "cli_feature_alpha",
|
||||
"--output-dir", str(alpha_dir),
|
||||
])
|
||||
alpha_path = alpha_dir / "cli_feature_alpha.pq"
|
||||
assert "Loaded data:" in alpha_result.output
|
||||
assert "Saved alpha:" in alpha_result.output
|
||||
assert "Weight stats" in alpha_result.output
|
||||
assert alpha_path.exists()
|
||||
assert not pd.read_parquet(alpha_path).empty
|
||||
|
||||
alpha_report_dir = tmp_path / "alpha_reports"
|
||||
alpha_eval_result = _invoke_ok(runner, [
|
||||
"alpha", "eval",
|
||||
"--alpha-path", str(alpha_path),
|
||||
"--data-path", str(daily_path),
|
||||
"--report-dir", str(alpha_report_dir),
|
||||
])
|
||||
alpha_report_path = alpha_report_dir / "cli_feature_alpha_eval.json"
|
||||
assert "ALPHA EVALUATION" in alpha_eval_result.output
|
||||
assert "Report saved:" in alpha_eval_result.output
|
||||
assert alpha_report_path.exists()
|
||||
|
||||
combo_dir = tmp_path / "combos"
|
||||
combo_result = _invoke_ok(runner, [
|
||||
"combo", "combine",
|
||||
"--alpha-paths", f"{alpha_path},{alpha_path}",
|
||||
"--combo-name", "cli_combo",
|
||||
"--method", "equal_weight",
|
||||
"--output-dir", str(combo_dir),
|
||||
])
|
||||
combo_path = combo_dir / "cli_combo.pq"
|
||||
assert "Saved combo:" in combo_result.output
|
||||
assert "Weight stats" in combo_result.output
|
||||
assert combo_path.exists()
|
||||
|
||||
portfolio_dir = tmp_path / "portfolio"
|
||||
build_result = _invoke_ok(runner, [
|
||||
"portfolio", "build",
|
||||
"--weights-path", str(combo_path),
|
||||
"--data-path", str(daily_path),
|
||||
"--booksize", "2000000",
|
||||
"--portfolio-name", "cli_portfolio",
|
||||
"--output-dir", str(portfolio_dir),
|
||||
])
|
||||
positions_path = portfolio_dir / "cli_portfolio.pq"
|
||||
assert "Saved positions:" in build_result.output
|
||||
assert "Gross exposure" in build_result.output
|
||||
assert positions_path.exists()
|
||||
|
||||
execution_dir = tmp_path / "execution"
|
||||
simulate_result = _invoke_ok(runner, [
|
||||
"portfolio", "simulate",
|
||||
"--positions-path", str(positions_path),
|
||||
"--data-path", str(daily_path),
|
||||
"--constraint", "suspension",
|
||||
"--constraint", "price_limit",
|
||||
"--constraint", "volume_cap",
|
||||
"--cost-bps", "5",
|
||||
"--slippage-bps", "5",
|
||||
"--volume-frac", "0.02",
|
||||
"--output-dir", str(execution_dir),
|
||||
])
|
||||
fills_path = execution_dir / "fills" / "cli_portfolio.pq"
|
||||
pnl_path = execution_dir / "pnl" / "cli_portfolio.pq"
|
||||
assert "Saved fills:" in simulate_result.output
|
||||
assert "Saved pnl:" in simulate_result.output
|
||||
assert "Total PnL:" in simulate_result.output
|
||||
assert fills_path.exists()
|
||||
assert pnl_path.exists()
|
||||
|
||||
eval_result = _invoke_ok(runner, [
|
||||
"portfolio", "eval",
|
||||
"--positions-path", str(positions_path),
|
||||
"--data-path", str(daily_path),
|
||||
])
|
||||
assert "Research-portfolio metrics:" in eval_result.output
|
||||
assert "cumulative_return" in eval_result.output
|
||||
assert "fitness" in eval_result.output
|
||||
|
||||
pqcat_result = _invoke_ok(runner, [
|
||||
"pqcat",
|
||||
str(positions_path),
|
||||
"--info",
|
||||
])
|
||||
assert "shape:" in pqcat_result.output
|
||||
assert "dtypes:" in pqcat_result.output
|
||||
assert "position_shares" in pqcat_result.output
|
||||
|
||||
alphaview_result = _invoke_ok(runner, [
|
||||
"alphaview",
|
||||
"--data-path", str(daily_path),
|
||||
"--alpha-path", str(alpha_path),
|
||||
"--symbol", "sh600000",
|
||||
"--start-date", "2024-01-02",
|
||||
"--end-date", "2024-01-12",
|
||||
"--columns", "close,volume",
|
||||
])
|
||||
assert "symbol: sh600000" in alphaview_result.output
|
||||
assert "cli_feature_alpha" in alphaview_result.output
|
||||
|
||||
|
||||
def test_cli_pipeline_accepts_partitioned_daily_dataset(tmp_path):
|
||||
runner = CliRunner()
|
||||
daily_bars = make_generated_daily_bars(include_missing=False)
|
||||
dataset_dir = tmp_path / "daily_dataset"
|
||||
dataset_frame = daily_bars.copy()
|
||||
dataset_frame["month"] = dataset_frame["date"].dt.strftime("%Y-%m")
|
||||
dataset_frame.to_parquet(dataset_dir, partition_cols=["month"], index=False)
|
||||
|
||||
alpha_dir = tmp_path / "alphas"
|
||||
alpha_result = _invoke_ok(runner, [
|
||||
"alpha", "compute",
|
||||
"--data-path", str(dataset_dir),
|
||||
"--alpha-type", "reversal",
|
||||
"--alpha-name", "dataset_reversal",
|
||||
"--lookback", "3",
|
||||
"--output-dir", str(alpha_dir),
|
||||
])
|
||||
alpha_path = alpha_dir / "dataset_reversal.pq"
|
||||
assert "Loaded data:" in alpha_result.output
|
||||
assert alpha_path.exists()
|
||||
|
||||
combo_dir = tmp_path / "combos"
|
||||
_invoke_ok(runner, [
|
||||
"combo", "combine",
|
||||
"--alpha-paths", str(alpha_path),
|
||||
"--combo-name", "dataset_combo",
|
||||
"--output-dir", str(combo_dir),
|
||||
])
|
||||
combo_path = combo_dir / "dataset_combo.pq"
|
||||
assert combo_path.exists()
|
||||
|
||||
portfolio_dir = tmp_path / "portfolio"
|
||||
_invoke_ok(runner, [
|
||||
"portfolio", "build",
|
||||
"--weights-path", str(combo_path),
|
||||
"--data-path", str(dataset_dir),
|
||||
"--booksize", "1000000",
|
||||
"--portfolio-name", "dataset_portfolio",
|
||||
"--output-dir", str(portfolio_dir),
|
||||
])
|
||||
positions_path = portfolio_dir / "dataset_portfolio.pq"
|
||||
assert positions_path.exists()
|
||||
|
||||
execution_dir = tmp_path / "execution"
|
||||
simulate_result = _invoke_ok(runner, [
|
||||
"portfolio", "simulate",
|
||||
"--positions-path", str(positions_path),
|
||||
"--data-path", str(dataset_dir),
|
||||
"--constraint", "suspension",
|
||||
"--output-dir", str(execution_dir),
|
||||
])
|
||||
assert "Saved fills:" in simulate_result.output
|
||||
assert (execution_dir / "fills" / "dataset_portfolio.pq").exists()
|
||||
assert (execution_dir / "pnl" / "dataset_portfolio.pq").exists()
|
||||
|
||||
|
||||
def test_cli_liquid_universe_masks_to_top_liquid_names(tmp_path):
|
||||
runner = CliRunner()
|
||||
daily_bars = make_generated_daily_bars(n_sessions=75, include_missing=False)
|
||||
daily_path = tmp_path / "daily_bars_75d.pq"
|
||||
daily_bars.to_parquet(daily_path, index=False)
|
||||
|
||||
alpha_dir = tmp_path / "alphas"
|
||||
result = _invoke_ok(runner, [
|
||||
"alpha", "compute",
|
||||
"--data-path", str(daily_path),
|
||||
"--alpha-type", "reversal_rank",
|
||||
"--alpha-name", "liquid_rank",
|
||||
"--lookback", "3",
|
||||
"--liquid-universe",
|
||||
"--universe-top-n", "2",
|
||||
"--output-dir", str(alpha_dir),
|
||||
])
|
||||
|
||||
alpha_path = alpha_dir / "liquid_rank.pq"
|
||||
alpha = pd.read_parquet(alpha_path)
|
||||
nonzero = alpha[alpha["weight"] != 0.0]
|
||||
assert "Saved alpha:" in result.output
|
||||
assert alpha_path.exists()
|
||||
assert not nonzero.empty
|
||||
assert nonzero.groupby("date")["symbol_id"].nunique().max() <= 2
|
||||
|
||||
|
||||
def test_cli_real_fixture_round_trips_through_portfolio(tmp_path):
|
||||
runner = CliRunner()
|
||||
|
||||
alpha_dir = tmp_path / "alphas"
|
||||
_invoke_ok(runner, [
|
||||
"alpha", "compute",
|
||||
"--data-path", str(FIXTURE_PATH),
|
||||
"--alpha-type", "reversal_vol",
|
||||
"--alpha-name", "real_cli_reversal_vol",
|
||||
"--lookback", "3",
|
||||
"--vol-window", "3",
|
||||
"--output-dir", str(alpha_dir),
|
||||
])
|
||||
alpha_path = alpha_dir / "real_cli_reversal_vol.pq"
|
||||
assert alpha_path.exists()
|
||||
assert not pd.read_parquet(alpha_path).empty
|
||||
|
||||
combo_dir = tmp_path / "combos"
|
||||
_invoke_ok(runner, [
|
||||
"combo", "combine",
|
||||
"--alpha-paths", str(alpha_path),
|
||||
"--combo-name", "real_cli_combo",
|
||||
"--output-dir", str(combo_dir),
|
||||
])
|
||||
combo_path = combo_dir / "real_cli_combo.pq"
|
||||
assert combo_path.exists()
|
||||
|
||||
portfolio_dir = tmp_path / "portfolio"
|
||||
_invoke_ok(runner, [
|
||||
"portfolio", "build",
|
||||
"--weights-path", str(combo_path),
|
||||
"--data-path", str(FIXTURE_PATH),
|
||||
"--booksize", "1000000",
|
||||
"--portfolio-name", "real_cli_portfolio",
|
||||
"--output-dir", str(portfolio_dir),
|
||||
])
|
||||
positions_path = portfolio_dir / "real_cli_portfolio.pq"
|
||||
positions = pd.read_parquet(positions_path)
|
||||
assert not positions.empty
|
||||
|
||||
eval_result = _invoke_ok(runner, [
|
||||
"portfolio", "eval",
|
||||
"--positions-path", str(positions_path),
|
||||
"--data-path", str(FIXTURE_PATH),
|
||||
])
|
||||
assert "Research-portfolio metrics:" in eval_result.output
|
||||
|
||||
|
||||
def test_cli_error_paths_are_clear_for_bad_user_inputs(tmp_path):
|
||||
runner = CliRunner()
|
||||
daily_bars = make_generated_daily_bars()
|
||||
daily_path = tmp_path / "daily_bars.pq"
|
||||
daily_bars.to_parquet(daily_path, index=False)
|
||||
|
||||
unknown_alpha = _invoke_error(runner, [
|
||||
"alpha", "compute",
|
||||
"--data-path", str(daily_path),
|
||||
"--alpha-type", "does_not_exist",
|
||||
"--alpha-name", "bad",
|
||||
"--output-dir", str(tmp_path / "alphas"),
|
||||
])
|
||||
assert "Unknown alpha-type" in unknown_alpha.output
|
||||
|
||||
malformed_param = _invoke_error(runner, [
|
||||
"alpha", "compute",
|
||||
"--data-path", str(daily_path),
|
||||
"--alpha-type", "reversal",
|
||||
"--alpha-name", "bad_param",
|
||||
"--param", "not-an-assignment",
|
||||
"--output-dir", str(tmp_path / "alphas"),
|
||||
])
|
||||
assert "--param must be name=value" in malformed_param.output
|
||||
|
||||
unknown_derived = _invoke_error(runner, [
|
||||
"derived", "compute",
|
||||
"--daily-path", str(daily_path),
|
||||
"--derived-type", "does_not_exist",
|
||||
"--derived-name", "bad",
|
||||
"--output-dir", str(tmp_path / "derived"),
|
||||
])
|
||||
assert "Unknown derived-type" in unknown_derived.output
|
||||
|
||||
bad_constraint_positions = pd.DataFrame({
|
||||
"symbol_id": ["sh600000"],
|
||||
"date": [pd.Timestamp("2024-01-02")],
|
||||
"portfolio_name": ["bad_constraint"],
|
||||
"target_weight": [1.0],
|
||||
"target_value": [1000.0],
|
||||
"target_shares": [100.0],
|
||||
"position_shares": [100],
|
||||
"position_value": [1000.0],
|
||||
"price": [10.0],
|
||||
})
|
||||
positions_path = tmp_path / "positions.pq"
|
||||
bad_constraint_positions.to_parquet(positions_path, index=False)
|
||||
unknown_constraint = _invoke_error(runner, [
|
||||
"portfolio", "simulate",
|
||||
"--positions-path", str(positions_path),
|
||||
"--data-path", str(daily_path),
|
||||
"--constraint", "not_a_constraint",
|
||||
"--output-dir", str(tmp_path / "execution"),
|
||||
])
|
||||
assert isinstance(unknown_constraint.exception, KeyError)
|
||||
assert "not_a_constraint" in str(unknown_constraint.exception)
|
||||
|
||||
pqcat_missing_column = _invoke_error(runner, [
|
||||
"pqcat",
|
||||
str(daily_path),
|
||||
"--columns", "close,not_a_column",
|
||||
])
|
||||
assert "Columns not found: not_a_column" in pqcat_missing_column.output
|
||||
|
||||
alphaview_missing_symbol = _invoke_error(runner, [
|
||||
"alphaview",
|
||||
"--data-path", str(daily_path),
|
||||
"--alpha-path", str(positions_path),
|
||||
"--symbol", "sh999999",
|
||||
])
|
||||
assert "Symbol 'sh999999' not found" in alphaview_missing_symbol.output
|
||||
@@ -1,6 +1,16 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from data.downloader import download_daily
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.network,
|
||||
pytest.mark.skipif(
|
||||
os.environ.get("CEQ_RUN_LIVE_DOWNLOADER") != "1",
|
||||
reason="set CEQ_RUN_LIVE_DOWNLOADER=1 to run live baostock/akshare smoke tests",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def test_download_single_stock():
|
||||
"""Smoke test: download data for 浦发银行 for a short window."""
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
"""Offline downloader contract tests with mocked data providers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
import data.downloader as downloader
|
||||
import pipeline.data.downloader as pipeline_downloader
|
||||
from data.downloader import download_daily, download_daily_batch
|
||||
from pipeline.common.schema import DATA_COLUMNS
|
||||
from pipeline.data.downloader import download_universe
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
def __init__(self, rows, error_code="0", error_msg=""):
|
||||
self.rows = rows
|
||||
self.error_code = error_code
|
||||
self.error_msg = error_msg
|
||||
self._idx = -1
|
||||
|
||||
def next(self):
|
||||
self._idx += 1
|
||||
return self._idx < len(self.rows)
|
||||
|
||||
def get_row_data(self):
|
||||
return self.rows[self._idx]
|
||||
|
||||
|
||||
def _daily_batch_row(
|
||||
date: str = "2024-01-02",
|
||||
open_: str = "10",
|
||||
high: str = "11",
|
||||
low: str = "9",
|
||||
close: str = "10.5",
|
||||
preclose: str = "10",
|
||||
volume: str = "1000",
|
||||
amount: str = "10500",
|
||||
) -> list[str]:
|
||||
return [
|
||||
date,
|
||||
open_,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
preclose,
|
||||
volume,
|
||||
amount,
|
||||
"1.23",
|
||||
"5.0",
|
||||
"1",
|
||||
"0",
|
||||
"8.0",
|
||||
"1.1",
|
||||
"2.2",
|
||||
"3.3",
|
||||
]
|
||||
|
||||
|
||||
def test_download_daily_uses_baostock_before_akshare_in_auto(monkeypatch):
|
||||
calls: list[str] = []
|
||||
expected = pd.DataFrame({
|
||||
"symbol": ["sh600000"],
|
||||
"date": ["2024-01-02"],
|
||||
"open": [10.0],
|
||||
"high": [11.0],
|
||||
"low": [9.0],
|
||||
"close": [10.5],
|
||||
"volume": [1000.0],
|
||||
"amount": [10500.0],
|
||||
})
|
||||
|
||||
def fake_baostock(symbol, start, end, adjust):
|
||||
calls.append("baostock")
|
||||
return expected
|
||||
|
||||
def fake_akshare(symbol, start, end, adjust):
|
||||
calls.append("akshare")
|
||||
raise AssertionError("akshare should not be called after baostock succeeds")
|
||||
|
||||
monkeypatch.setattr(downloader, "_download_baostock", fake_baostock)
|
||||
monkeypatch.setattr(downloader, "_download_akshare", fake_akshare)
|
||||
|
||||
result = download_daily("sh600000", "2024-01-02", "2024-01-02", source="auto")
|
||||
|
||||
assert calls == ["baostock"]
|
||||
assert result["date"].tolist() == [pd.Timestamp("2024-01-02")]
|
||||
assert result["close"].tolist() == [10.5]
|
||||
|
||||
|
||||
def test_download_daily_falls_back_to_akshare_when_baostock_empty(monkeypatch):
|
||||
calls: list[str] = []
|
||||
fallback = pd.DataFrame({
|
||||
"symbol": ["sz000001"],
|
||||
"date": ["2024-01-02"],
|
||||
"open": [20.0],
|
||||
"high": [21.0],
|
||||
"low": [19.0],
|
||||
"close": [20.5],
|
||||
"volume": [2000.0],
|
||||
"amount": [41000.0],
|
||||
})
|
||||
|
||||
monkeypatch.setattr(
|
||||
downloader,
|
||||
"_download_baostock",
|
||||
lambda symbol, start, end, adjust: calls.append("baostock") or None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
downloader,
|
||||
"_download_akshare",
|
||||
lambda symbol, start, end, adjust: calls.append("akshare") or fallback,
|
||||
)
|
||||
|
||||
result = download_daily("sz000001", "2024-01-02", "2024-01-02", source="auto")
|
||||
|
||||
assert calls == ["baostock", "akshare"]
|
||||
assert result["symbol"].tolist() == ["sz000001"]
|
||||
assert result["date"].tolist() == [pd.Timestamp("2024-01-02")]
|
||||
|
||||
|
||||
def test_download_daily_batch_maps_rich_schema_and_vwap(monkeypatch):
|
||||
query_calls: list[dict] = []
|
||||
login_count = 0
|
||||
logout_count = 0
|
||||
|
||||
def fake_login():
|
||||
nonlocal login_count
|
||||
login_count += 1
|
||||
|
||||
def fake_logout():
|
||||
nonlocal logout_count
|
||||
logout_count += 1
|
||||
|
||||
def fake_query(**kwargs):
|
||||
query_calls.append(kwargs)
|
||||
rows = [
|
||||
_daily_batch_row(volume="1000", amount="10500"),
|
||||
_daily_batch_row(date="2024-01-03", volume="0", amount="0"),
|
||||
]
|
||||
return _FakeResult(rows)
|
||||
|
||||
monkeypatch.setattr(downloader.bs, "login", fake_login)
|
||||
monkeypatch.setattr(downloader.bs, "logout", fake_logout)
|
||||
monkeypatch.setattr(downloader.bs, "query_history_k_data_plus", fake_query)
|
||||
|
||||
[(symbol, frame)] = list(
|
||||
download_daily_batch(
|
||||
["sh600000"],
|
||||
"2024-01-02",
|
||||
"2024-01-03",
|
||||
adjust="hfq",
|
||||
)
|
||||
)
|
||||
|
||||
assert symbol == "sh600000"
|
||||
assert query_calls[0]["code"] == "sh.600000"
|
||||
assert query_calls[0]["adjustflag"] == "1"
|
||||
assert login_count == 1
|
||||
assert logout_count == 1
|
||||
assert frame is not None
|
||||
assert frame.columns.tolist() == [
|
||||
"symbol", "date", "open", "high", "low", "close", "preclose",
|
||||
"volume", "amount", "vwap", "turn", "pctChg", "tradestatus", "isST",
|
||||
"peTTM", "pbMRQ", "psTTM", "pcfNcfTTM",
|
||||
]
|
||||
assert np.isclose(frame["vwap"].iloc[0], 10.5)
|
||||
assert pd.isna(frame["vwap"].iloc[1])
|
||||
assert pd.api.types.is_datetime64_any_dtype(frame["date"])
|
||||
assert pd.api.types.is_numeric_dtype(frame["tradestatus"])
|
||||
|
||||
|
||||
def test_download_daily_batch_relogs_and_retries_session_loss(monkeypatch):
|
||||
responses = [
|
||||
_FakeResult([], error_code="10002007", error_msg="用户未登录"),
|
||||
_FakeResult([_daily_batch_row()]),
|
||||
]
|
||||
login_count = 0
|
||||
logout_count = 0
|
||||
|
||||
def fake_login():
|
||||
nonlocal login_count
|
||||
login_count += 1
|
||||
|
||||
def fake_logout():
|
||||
nonlocal logout_count
|
||||
logout_count += 1
|
||||
|
||||
monkeypatch.setattr(downloader.bs, "login", fake_login)
|
||||
monkeypatch.setattr(downloader.bs, "logout", fake_logout)
|
||||
monkeypatch.setattr(
|
||||
downloader.bs,
|
||||
"query_history_k_data_plus",
|
||||
lambda **kwargs: responses.pop(0),
|
||||
)
|
||||
|
||||
[(symbol, frame)] = list(download_daily_batch(["sh600000"], "2024-01-02", "2024-01-02"))
|
||||
|
||||
assert symbol == "sh600000"
|
||||
assert frame is not None
|
||||
assert len(frame) == 1
|
||||
assert login_count == 2
|
||||
assert logout_count == 2
|
||||
|
||||
|
||||
def test_download_daily_batch_uses_akshare_fallback_when_enabled(monkeypatch):
|
||||
fallback = pd.DataFrame({
|
||||
"symbol": ["sh600000"],
|
||||
"date": ["2024-01-02"],
|
||||
"open": [10.0],
|
||||
"high": [11.0],
|
||||
"low": [9.0],
|
||||
"close": [10.5],
|
||||
"volume": [1000.0],
|
||||
"amount": [10500.0],
|
||||
})
|
||||
|
||||
monkeypatch.setattr(downloader.bs, "login", lambda: None)
|
||||
monkeypatch.setattr(downloader.bs, "logout", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
downloader.bs,
|
||||
"query_history_k_data_plus",
|
||||
lambda **kwargs: _FakeResult([], error_code="1", error_msg="no data"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
downloader,
|
||||
"_download_akshare",
|
||||
lambda symbol, start, end, adjust: fallback.copy(),
|
||||
)
|
||||
|
||||
[(symbol, frame)] = list(
|
||||
download_daily_batch(
|
||||
["sh600000"],
|
||||
"2024-01-02",
|
||||
"2024-01-02",
|
||||
akshare_fallback=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert symbol == "sh600000"
|
||||
assert frame is not None
|
||||
assert frame["date"].tolist() == [pd.Timestamp("2024-01-02")]
|
||||
assert frame["close"].tolist() == [10.5]
|
||||
|
||||
|
||||
def test_download_universe_writes_daily_partitions_from_mock_batch(tmp_path, monkeypatch):
|
||||
batch_frame = pd.DataFrame({
|
||||
"symbol": ["sh600000", "sh600000"],
|
||||
"date": pd.to_datetime(["2024-01-02", "2024-02-01"]),
|
||||
"open": [10.0, 11.0],
|
||||
"high": [11.0, 12.0],
|
||||
"low": [9.0, 10.0],
|
||||
"close": [10.5, 11.5],
|
||||
"preclose": [10.0, 10.5],
|
||||
"volume": [1000.0, 1200.0],
|
||||
"amount": [10500.0, 13800.0],
|
||||
"vwap": [10.5, 11.5],
|
||||
"turn": [1.0, 1.1],
|
||||
"pctChg": [5.0, 9.5],
|
||||
"tradestatus": [1, 1],
|
||||
"isST": [0, 0],
|
||||
"peTTM": [8.0, 8.1],
|
||||
"pbMRQ": [1.1, 1.2],
|
||||
"psTTM": [2.1, 2.2],
|
||||
"pcfNcfTTM": [3.1, 3.2],
|
||||
})
|
||||
|
||||
monkeypatch.setattr(
|
||||
pipeline_downloader,
|
||||
"_resolve_universe",
|
||||
lambda universe, max_symbols=0: pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sz000001"],
|
||||
"symbol_name": ["PF Bank", "Ping An Bank"],
|
||||
}),
|
||||
)
|
||||
|
||||
def fake_batch(symbols, start, end, adjust="qfq"):
|
||||
assert symbols == ["sh600000", "sz000001"]
|
||||
assert adjust == "qfq"
|
||||
yield "sh600000", batch_frame
|
||||
yield "sz000001", None
|
||||
|
||||
monkeypatch.setattr(pipeline_downloader, "download_daily_batch", fake_batch)
|
||||
|
||||
stats = download_universe(
|
||||
universe="toy",
|
||||
start_date="2024-01-02",
|
||||
end_date="2024-02-01",
|
||||
output_dir=str(tmp_path),
|
||||
chunk_size=1,
|
||||
)
|
||||
|
||||
dataset_path = tmp_path / "toy"
|
||||
written = pd.read_parquet(dataset_path).sort_values(["date", "symbol_id"]).reset_index(drop=True)
|
||||
assert stats == {
|
||||
"dataset_path": str(dataset_path),
|
||||
"n_symbols": 1,
|
||||
"n_requested": 2,
|
||||
"n_rows": 2,
|
||||
"date_min": "2024-01-02",
|
||||
"date_max": "2024-02-01",
|
||||
}
|
||||
assert (dataset_path / "month=2024-01").exists()
|
||||
assert (dataset_path / "month=2024-02").exists()
|
||||
assert written[DATA_COLUMNS].columns.tolist() == DATA_COLUMNS
|
||||
assert written["symbol_name"].tolist() == ["PF Bank", "PF Bank"]
|
||||
@@ -13,6 +13,7 @@ from pipeline.features.registry import (
|
||||
get_feature,
|
||||
load_feature_module,
|
||||
)
|
||||
from pipeline.derived.compute import compute_derived
|
||||
|
||||
|
||||
def _minute_bars() -> pd.DataFrame:
|
||||
@@ -77,6 +78,27 @@ def test_minute_daily_summary_feature_preserves_legacy_positional_compute():
|
||||
pd.testing.assert_frame_equal(direct, via_registry)
|
||||
|
||||
|
||||
def test_legacy_feature_compute_matches_canonical_derived_compute():
|
||||
daily = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sz000001", "sh600000"],
|
||||
"date": pd.to_datetime(["2024-01-02", "2024-01-02", "2024-01-03"]),
|
||||
"close": [11.0, 20.5, 12.0],
|
||||
})
|
||||
|
||||
legacy_feature = compute_feature(
|
||||
minute=_minute_bars(),
|
||||
daily=daily,
|
||||
feature_type="minute_daily_summary",
|
||||
)
|
||||
canonical_derived = compute_derived(
|
||||
"minute_daily_summary",
|
||||
daily=daily,
|
||||
minute=_minute_bars(),
|
||||
)
|
||||
|
||||
pd.testing.assert_frame_equal(legacy_feature, canonical_derived)
|
||||
|
||||
|
||||
def test_load_external_feature_module_and_filter_params(tmp_path):
|
||||
module_path = tmp_path / "external_feature.py"
|
||||
module_path.write_text(textwrap.dedent('''
|
||||
|
||||
@@ -306,6 +306,47 @@ def test_construct_positions_schema():
|
||||
assert pos["position_shares"].dtype == np.int64
|
||||
|
||||
|
||||
def test_construct_positions_empty_weights_returns_schema():
|
||||
data = _make_data(n_days=3)
|
||||
empty_weights = pd.DataFrame(columns=["symbol_id", "date", "combo_name", "weight"])
|
||||
|
||||
pos = construct_positions(
|
||||
empty_weights,
|
||||
data,
|
||||
booksize=1e6,
|
||||
portfolio_name="empty",
|
||||
)
|
||||
|
||||
assert list(pos.columns) == POSITION_COLUMNS
|
||||
assert pos.empty
|
||||
|
||||
|
||||
def test_construct_positions_ignores_absent_or_bad_prices():
|
||||
dates = pd.to_datetime(["2024-01-01", "2024-01-02"])
|
||||
data = pd.DataFrame([
|
||||
{"symbol_id": "sh600000", "date": dates[0], "close": np.nan, "isST": 0},
|
||||
{"symbol_id": "sz000001", "date": dates[0], "close": 20.0, "isST": 0},
|
||||
{"symbol_id": "sh600000", "date": dates[1], "close": 10.0, "isST": 0},
|
||||
{"symbol_id": "sz000001", "date": dates[1], "close": 20.0, "isST": 0},
|
||||
])
|
||||
weights = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sz000001", "sh600000", "sz000001"],
|
||||
"date": [dates[0], dates[0], dates[1], dates[1]],
|
||||
"combo_name": ["combo"] * 4,
|
||||
"weight": [1.0, -1.0, 1.0, -1.0],
|
||||
})
|
||||
|
||||
pos = construct_positions(weights, data, booksize=10000.0, portfolio_name="bad_price")
|
||||
|
||||
bad_price_rows = pos[
|
||||
(pos["date"] == dates[0])
|
||||
& (pos["symbol_id"] == "sh600000")
|
||||
]
|
||||
assert bad_price_rows.empty or (bad_price_rows["target_weight"] == 0.0).all()
|
||||
assert np.isfinite(pos["target_value"]).all()
|
||||
assert np.isfinite(pos["position_value"]).all()
|
||||
|
||||
|
||||
def test_construct_positions_threads_state_and_closes_absent():
|
||||
data = _make_data()
|
||||
weights = _make_weights(data)
|
||||
@@ -321,6 +362,34 @@ def test_construct_positions_threads_state_and_closes_absent():
|
||||
assert final.empty or (final["position_shares"] == 0).all()
|
||||
|
||||
|
||||
def test_construct_positions_closes_absent_short_position():
|
||||
dates = pd.to_datetime(["2024-01-02", "2024-01-03"])
|
||||
data = pd.DataFrame([
|
||||
{"symbol_id": sym, "date": d, "close": price, "isST": 0}
|
||||
for d in dates
|
||||
for sym, price in (("sh600000", 10.0), ("sz000001", 20.0))
|
||||
])
|
||||
weights = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sz000001", "sz000001"],
|
||||
"date": [dates[0], dates[0], dates[1]],
|
||||
"combo_name": ["combo", "combo", "combo"],
|
||||
"weight": [-1.0, 1.0, 1.0],
|
||||
})
|
||||
|
||||
pos = construct_positions(weights, data, booksize=20000.0, portfolio_name="absent_short")
|
||||
|
||||
first_day_short = pos[
|
||||
(pos["date"] == dates[0])
|
||||
& (pos["symbol_id"] == "sh600000")
|
||||
]
|
||||
final_day_short = pos[
|
||||
(pos["date"] == dates[1])
|
||||
& (pos["symbol_id"] == "sh600000")
|
||||
]
|
||||
assert (first_day_short["position_shares"] < 0).all()
|
||||
assert final_day_short.empty or (final_day_short["position_shares"] == 0).all()
|
||||
|
||||
|
||||
def test_construct_positions_carries_book_on_zero_gross(caplog):
|
||||
dates = pd.to_datetime(["2024-01-01", "2024-01-02"])
|
||||
symbols = ["sh600000", "sz000001"]
|
||||
@@ -392,6 +461,42 @@ def test_volume_cap_uses_traded_value():
|
||||
assert low[0] == -10000.0
|
||||
|
||||
|
||||
def test_constraints_compose_repeatably_regardless_of_order():
|
||||
n = 1
|
||||
sl = _slice(
|
||||
n,
|
||||
tradestatus=np.array([0.0]),
|
||||
limit_status=np.array([LimitStatus.UP_LIMIT.value], dtype=np.int8),
|
||||
amount=np.array([1_000.0]),
|
||||
price=np.array([10.0]),
|
||||
)
|
||||
ctx = TradeContext(np.zeros(n, np.int64), np.array([500]), sl, 1e6)
|
||||
first_order = ReferenceSimulator(
|
||||
constraints=[
|
||||
SuspensionConstraint(),
|
||||
PriceLimitConstraint(),
|
||||
VolumeCapConstraint(max_frac=0.1),
|
||||
],
|
||||
cost_bps=10,
|
||||
).fill(ctx)
|
||||
reversed_order = ReferenceSimulator(
|
||||
constraints=[
|
||||
VolumeCapConstraint(max_frac=0.1),
|
||||
PriceLimitConstraint(),
|
||||
SuspensionConstraint(),
|
||||
],
|
||||
cost_bps=10,
|
||||
).fill(ctx)
|
||||
|
||||
assert first_order.traded_shares.tolist() == [0]
|
||||
assert first_order.realized_shares.tolist() == [0]
|
||||
assert first_order.blocked.tolist() == [1]
|
||||
assert np.array_equal(first_order.traded_shares, reversed_order.traded_shares)
|
||||
assert np.array_equal(first_order.realized_shares, reversed_order.realized_shares)
|
||||
assert np.array_equal(first_order.blocked, reversed_order.blocked)
|
||||
assert np.array_equal(first_order.cost, reversed_order.cost)
|
||||
|
||||
|
||||
# --- ReferenceSimulator ------------------------------------------------------
|
||||
|
||||
def test_simulator_next_open_and_blocked_buy_holds_prev():
|
||||
@@ -474,6 +579,91 @@ def test_simulator_cost_only_on_nonzero_realized_trades():
|
||||
assert np.isclose(res.cost[1], 50 * 20 * 10 / 1e4)
|
||||
|
||||
|
||||
def test_simulator_short_to_long_flip_trades_full_delta():
|
||||
dates = pd.to_datetime(["2024-01-01", "2024-01-02", "2024-01-03"])
|
||||
positions = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sh600000"],
|
||||
"date": [dates[0], dates[1]],
|
||||
"portfolio_name": ["flip", "flip"],
|
||||
"target_weight": [-1.0, 1.0],
|
||||
"target_value": [-1000.0, 1000.0],
|
||||
"target_shares": [-100.0, 100.0],
|
||||
"position_shares": [-100, 100],
|
||||
"position_value": [-1000.0, 1000.0],
|
||||
"price": [10.0, 10.0],
|
||||
})
|
||||
data = pd.DataFrame({
|
||||
"symbol_id": ["sh600000"] * 3,
|
||||
"date": dates,
|
||||
"open": [10.0, 10.0, 10.0],
|
||||
"close": [10.0, 10.0, 10.0],
|
||||
"preclose": [10.0, 10.0, 10.0],
|
||||
"amount": [1e9, 1e9, 1e9],
|
||||
"tradestatus": [1, 1, 1],
|
||||
"isST": [0, 0, 0],
|
||||
})
|
||||
|
||||
fills, _ = ReferenceSimulator().run(positions, data)
|
||||
|
||||
by_date = fills.set_index("date")
|
||||
assert by_date.loc[dates[1], "traded_shares"] == -100
|
||||
assert by_date.loc[dates[1], "realized_shares"] == -100
|
||||
assert by_date.loc[dates[2], "prev_shares"] == -100
|
||||
assert by_date.loc[dates[2], "traded_shares"] == 200
|
||||
assert by_date.loc[dates[2], "realized_shares"] == 100
|
||||
|
||||
|
||||
def test_simulator_volume_cap_partially_fills_sell():
|
||||
sl = _slice(1, amount=np.array([10_000.0]), price=np.array([10.0]))
|
||||
ctx = TradeContext(
|
||||
np.array([1000], np.int64),
|
||||
np.array([0], np.int64),
|
||||
sl,
|
||||
1_000_000.0,
|
||||
)
|
||||
|
||||
result = ReferenceSimulator(
|
||||
constraints=[VolumeCapConstraint(max_frac=0.10)]
|
||||
).fill(ctx)
|
||||
|
||||
assert result.traded_shares.tolist() == [-100]
|
||||
assert result.realized_shares.tolist() == [900]
|
||||
assert result.blocked.tolist() == [1]
|
||||
|
||||
|
||||
def test_simulator_missing_next_open_has_zero_cost_and_turnover():
|
||||
dates = pd.to_datetime(["2024-01-01", "2024-01-02"])
|
||||
positions = pd.DataFrame({
|
||||
"symbol_id": ["sh600000"],
|
||||
"date": [dates[0]],
|
||||
"portfolio_name": ["missing_open"],
|
||||
"target_weight": [1.0],
|
||||
"target_value": [1000.0],
|
||||
"target_shares": [100.0],
|
||||
"position_shares": [100],
|
||||
"position_value": [1000.0],
|
||||
"price": [10.0],
|
||||
})
|
||||
data = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sh600000"],
|
||||
"date": dates,
|
||||
"open": [10.0, np.nan],
|
||||
"close": [10.0, 10.0],
|
||||
"preclose": [10.0, 10.0],
|
||||
"amount": [1e9, 1e9],
|
||||
"tradestatus": [1, 1],
|
||||
"isST": [0, 0],
|
||||
})
|
||||
|
||||
fills, pnl = ReferenceSimulator(cost_bps=10, slippage_bps=5).run(positions, data)
|
||||
|
||||
assert fills["traded_shares"].iloc[0] == 100
|
||||
assert fills["trade_cost"].iloc[0] == 0.0
|
||||
assert pnl["cost"].iloc[0] == 0.0
|
||||
assert pnl["turnover"].iloc[0] == 0.0
|
||||
assert pnl["gross_exposure"].iloc[0] == 1000.0
|
||||
|
||||
|
||||
def test_simple_cost_model_adds_cost_and_slippage_without_price_adjustment():
|
||||
model = SimpleProportionalCostModel(cost_bps=10, slippage_bps=5)
|
||||
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Malformed parquet/input tests for phase boundary contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from pipeline.alpha.compute import compute_alpha
|
||||
from pipeline.combo.combine import combine_alphas
|
||||
from pipeline.derived.compute import validate_derived_frame
|
||||
from pipeline.portfolio.construct import construct_positions
|
||||
from pipeline.portfolio.simulator import ReferenceSimulator
|
||||
from tests.helpers import make_generated_daily_bars
|
||||
|
||||
|
||||
def test_alpha_compute_rejects_daily_data_without_close():
|
||||
daily = make_generated_daily_bars().drop(columns=["close"])
|
||||
|
||||
with pytest.raises(KeyError, match="close"):
|
||||
compute_alpha(daily, "bad", "reversal", lookback=3)
|
||||
|
||||
|
||||
def test_alpha_feature_path_rejects_duplicate_symbol_dates(tmp_path):
|
||||
daily = make_generated_daily_bars()
|
||||
feature = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sh600000"],
|
||||
"date": ["2024-01-02 09:30:00", "2024-01-02 15:00:00"],
|
||||
"toy_feature": [1.0, 2.0],
|
||||
})
|
||||
feature_path = tmp_path / "duplicate_feature_keys.pq"
|
||||
feature.to_parquet(feature_path, index=False)
|
||||
|
||||
with pytest.raises(ValueError, match="duplicate symbol_id,date"):
|
||||
compute_alpha(
|
||||
daily,
|
||||
"bad_features",
|
||||
"reversal",
|
||||
lookback=3,
|
||||
feature_paths=[str(feature_path)],
|
||||
)
|
||||
|
||||
|
||||
def test_derived_validation_rejects_bool_value_columns():
|
||||
derived = pd.DataFrame({
|
||||
"symbol_id": ["sh600000"],
|
||||
"date": [pd.Timestamp("2024-01-02")],
|
||||
"is_good": [True],
|
||||
})
|
||||
|
||||
with pytest.raises(ValueError, match="numeric"):
|
||||
validate_derived_frame(derived)
|
||||
|
||||
|
||||
def test_combo_combine_rejects_missing_weight_column(tmp_path):
|
||||
bad_alpha = pd.DataFrame({
|
||||
"symbol_id": ["sh600000"],
|
||||
"date": [pd.Timestamp("2024-01-02")],
|
||||
"alpha_name": ["bad"],
|
||||
})
|
||||
bad_alpha_path = tmp_path / "bad_alpha.pq"
|
||||
bad_alpha.to_parquet(bad_alpha_path, index=False)
|
||||
|
||||
with pytest.raises(KeyError, match="weight"):
|
||||
combine_alphas([str(bad_alpha_path)], "bad_combo")
|
||||
|
||||
|
||||
def test_portfolio_build_rejects_weights_without_symbol_id():
|
||||
daily = make_generated_daily_bars()
|
||||
bad_weights = pd.DataFrame({
|
||||
"date": [pd.Timestamp("2024-01-02")],
|
||||
"combo_name": ["bad"],
|
||||
"weight": [1.0],
|
||||
})
|
||||
|
||||
with pytest.raises(KeyError, match="symbol_id"):
|
||||
construct_positions(
|
||||
bad_weights,
|
||||
daily,
|
||||
booksize=1_000_000.0,
|
||||
portfolio_name="bad_portfolio",
|
||||
)
|
||||
|
||||
|
||||
def test_portfolio_simulate_rejects_positions_without_position_shares():
|
||||
daily = make_generated_daily_bars()
|
||||
bad_positions = pd.DataFrame({
|
||||
"symbol_id": ["sh600000"],
|
||||
"date": [pd.Timestamp("2024-01-02")],
|
||||
"portfolio_name": ["bad"],
|
||||
"target_weight": [1.0],
|
||||
"target_value": [1000.0],
|
||||
"target_shares": [100.0],
|
||||
"position_value": [1000.0],
|
||||
"price": [10.0],
|
||||
})
|
||||
|
||||
with pytest.raises(KeyError, match="position_shares"):
|
||||
ReferenceSimulator().run(bad_positions, daily)
|
||||
@@ -0,0 +1,508 @@
|
||||
"""Verbose offline checks for the daily research workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from pipeline.alpha.compute import compute_alpha
|
||||
from pipeline.combo.combine import combine_alphas
|
||||
from pipeline.common.schema import (
|
||||
ALPHA_COLUMNS,
|
||||
COMBO_COLUMNS,
|
||||
FILL_COLUMNS,
|
||||
PNL_COLUMNS,
|
||||
POSITION_COLUMNS,
|
||||
)
|
||||
from pipeline.portfolio.constraints import (
|
||||
PriceLimitConstraint,
|
||||
SuspensionConstraint,
|
||||
VolumeCapConstraint,
|
||||
)
|
||||
from pipeline.portfolio.construct import construct_positions
|
||||
from pipeline.portfolio.research import evaluate_portfolio
|
||||
from pipeline.portfolio.simulator import ReferenceSimulator
|
||||
from tests.helpers import (
|
||||
GENERATED_SYMBOLS,
|
||||
generated_sessions,
|
||||
make_generated_alpha_weights,
|
||||
make_generated_combo_weights,
|
||||
make_generated_daily_bars,
|
||||
)
|
||||
|
||||
|
||||
FIXTURE_PATH = Path(__file__).parent / "fixtures" / "daily_bars_real_2024_01_sample.pq"
|
||||
|
||||
|
||||
def _assert_sorted_by_symbol_date(frame: pd.DataFrame) -> None:
|
||||
expected = frame.sort_values(["symbol_id", "date"]).reset_index(drop=True)
|
||||
pd.testing.assert_frame_equal(frame.reset_index(drop=True), expected)
|
||||
|
||||
|
||||
def _assert_metric_dict_is_finite(metrics: dict[str, float]) -> None:
|
||||
for key in (
|
||||
"cumulative_return",
|
||||
"sharpe_annual",
|
||||
"turnover_annual",
|
||||
"max_drawdown",
|
||||
"hit_rate",
|
||||
"n_dates",
|
||||
):
|
||||
assert key in metrics
|
||||
assert np.isfinite(metrics[key])
|
||||
assert "ic" not in metrics
|
||||
assert "rank_ic" not in metrics
|
||||
assert "ir" not in metrics
|
||||
|
||||
|
||||
def test_tiny_workflow_golden_outputs_are_stable(tmp_path):
|
||||
dates = pd.to_datetime(["2024-01-02", "2024-01-03", "2024-01-04"])
|
||||
daily_bars = pd.DataFrame([
|
||||
{
|
||||
"symbol_id": "sh600000",
|
||||
"symbol_name": "A",
|
||||
"date": dates[0],
|
||||
"open": 10.0,
|
||||
"high": 10.0,
|
||||
"low": 10.0,
|
||||
"close": 10.0,
|
||||
"preclose": 10.0,
|
||||
"volume": 1_000_000.0,
|
||||
"amount": 10_000_000.0,
|
||||
"vwap": 10.0,
|
||||
"turn": 1.0,
|
||||
"pctChg": 0.0,
|
||||
"tradestatus": 1,
|
||||
"isST": 0,
|
||||
"peTTM": 1.0,
|
||||
"pbMRQ": 1.0,
|
||||
"psTTM": 1.0,
|
||||
"pcfNcfTTM": 1.0,
|
||||
},
|
||||
{
|
||||
"symbol_id": "sz000001",
|
||||
"symbol_name": "B",
|
||||
"date": dates[0],
|
||||
"open": 20.0,
|
||||
"high": 20.0,
|
||||
"low": 20.0,
|
||||
"close": 20.0,
|
||||
"preclose": 20.0,
|
||||
"volume": 1_000_000.0,
|
||||
"amount": 20_000_000.0,
|
||||
"vwap": 20.0,
|
||||
"turn": 1.0,
|
||||
"pctChg": 0.0,
|
||||
"tradestatus": 1,
|
||||
"isST": 0,
|
||||
"peTTM": 1.0,
|
||||
"pbMRQ": 1.0,
|
||||
"psTTM": 1.0,
|
||||
"pcfNcfTTM": 1.0,
|
||||
},
|
||||
{
|
||||
"symbol_id": "sh600000",
|
||||
"symbol_name": "A",
|
||||
"date": dates[1],
|
||||
"open": 10.0,
|
||||
"high": 12.0,
|
||||
"low": 10.0,
|
||||
"close": 12.0,
|
||||
"preclose": 10.0,
|
||||
"volume": 1_000_000.0,
|
||||
"amount": 10_000_000.0,
|
||||
"vwap": 10.0,
|
||||
"turn": 1.0,
|
||||
"pctChg": 20.0,
|
||||
"tradestatus": 1,
|
||||
"isST": 0,
|
||||
"peTTM": 1.0,
|
||||
"pbMRQ": 1.0,
|
||||
"psTTM": 1.0,
|
||||
"pcfNcfTTM": 1.0,
|
||||
},
|
||||
{
|
||||
"symbol_id": "sz000001",
|
||||
"symbol_name": "B",
|
||||
"date": dates[1],
|
||||
"open": 20.0,
|
||||
"high": 20.0,
|
||||
"low": 18.0,
|
||||
"close": 18.0,
|
||||
"preclose": 20.0,
|
||||
"volume": 1_000_000.0,
|
||||
"amount": 20_000_000.0,
|
||||
"vwap": 20.0,
|
||||
"turn": 1.0,
|
||||
"pctChg": -10.0,
|
||||
"tradestatus": 1,
|
||||
"isST": 0,
|
||||
"peTTM": 1.0,
|
||||
"pbMRQ": 1.0,
|
||||
"psTTM": 1.0,
|
||||
"pcfNcfTTM": 1.0,
|
||||
},
|
||||
{
|
||||
"symbol_id": "sh600000",
|
||||
"symbol_name": "A",
|
||||
"date": dates[2],
|
||||
"open": 12.0,
|
||||
"high": 13.0,
|
||||
"low": 12.0,
|
||||
"close": 13.0,
|
||||
"preclose": 12.0,
|
||||
"volume": 1_000_000.0,
|
||||
"amount": 12_000_000.0,
|
||||
"vwap": 12.0,
|
||||
"turn": 1.0,
|
||||
"pctChg": 8.33,
|
||||
"tradestatus": 1,
|
||||
"isST": 0,
|
||||
"peTTM": 1.0,
|
||||
"pbMRQ": 1.0,
|
||||
"psTTM": 1.0,
|
||||
"pcfNcfTTM": 1.0,
|
||||
},
|
||||
{
|
||||
"symbol_id": "sz000001",
|
||||
"symbol_name": "B",
|
||||
"date": dates[2],
|
||||
"open": 18.0,
|
||||
"high": 21.0,
|
||||
"low": 18.0,
|
||||
"close": 21.0,
|
||||
"preclose": 18.0,
|
||||
"volume": 1_000_000.0,
|
||||
"amount": 18_000_000.0,
|
||||
"vwap": 18.0,
|
||||
"turn": 1.0,
|
||||
"pctChg": 16.67,
|
||||
"tradestatus": 1,
|
||||
"isST": 0,
|
||||
"peTTM": 1.0,
|
||||
"pbMRQ": 1.0,
|
||||
"psTTM": 1.0,
|
||||
"pcfNcfTTM": 1.0,
|
||||
},
|
||||
])
|
||||
alpha = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sz000001", "sh600000", "sz000001"],
|
||||
"date": [dates[0], dates[0], dates[1], dates[1]],
|
||||
"alpha_name": ["gold_alpha"] * 4,
|
||||
"weight": [1.0, -1.0, -1.0, 1.0],
|
||||
})
|
||||
alpha_path = tmp_path / "gold_alpha.pq"
|
||||
alpha.to_parquet(alpha_path, index=False)
|
||||
|
||||
combo = combine_alphas([str(alpha_path)], "gold_combo")
|
||||
positions = construct_positions(combo, daily_bars, booksize=20_000.0, portfolio_name="gold_port")
|
||||
fills, pnl = ReferenceSimulator().run(positions, daily_bars)
|
||||
|
||||
expected_combo = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sh600000", "sz000001", "sz000001"],
|
||||
"date": [dates[0], dates[1], dates[0], dates[1]],
|
||||
"combo_name": ["gold_combo"] * 4,
|
||||
"weight": [1.0, -1.0, -1.0, 1.0],
|
||||
})
|
||||
expected_positions = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sh600000", "sz000001", "sz000001"],
|
||||
"date": [dates[0], dates[1], dates[0], dates[1]],
|
||||
"portfolio_name": ["gold_port"] * 4,
|
||||
"target_weight": [0.5, -0.5, -0.5, 0.5],
|
||||
"target_value": [10000.0, -10000.0, -10000.0, 10000.0],
|
||||
"target_shares": [1000.0, -10000.0 / 12.0, -500.0, 10000.0 / 18.0],
|
||||
"position_shares": [1000, -833, -500, 556],
|
||||
"position_value": [10000.0, -9996.0, -10000.0, 10008.0],
|
||||
"price": [10.0, 12.0, 20.0, 18.0],
|
||||
})
|
||||
expected_fills = pd.DataFrame({
|
||||
"symbol_id": ["sh600000", "sz000001", "sh600000", "sz000001"],
|
||||
"date": [dates[1], dates[1], dates[2], dates[2]],
|
||||
"portfolio_name": ["gold_port"] * 4,
|
||||
"prev_shares": [0, 0, 1000, -500],
|
||||
"target_shares": [1000, -500, -833, 556],
|
||||
"traded_shares": [1000, -500, -1833, 1056],
|
||||
"realized_shares": [1000, -500, -833, 556],
|
||||
"blocked": [0, 0, 0, 0],
|
||||
"trade_cost": [0.0, 0.0, 0.0, 0.0],
|
||||
})
|
||||
expected_pnl = pd.DataFrame({
|
||||
"date": [dates[1], dates[2]],
|
||||
"portfolio_name": ["gold_port", "gold_port"],
|
||||
"gross_exposure": [21000.0, 22505.0],
|
||||
"net_exposure": [3000.0, 847.0],
|
||||
"pnl": [3000.0, 835.0],
|
||||
"cost": [0.0, 0.0],
|
||||
"turnover": [1.0, 2.0502],
|
||||
"n_positions": [2, 2],
|
||||
})
|
||||
|
||||
pd.testing.assert_frame_equal(combo, expected_combo)
|
||||
pd.testing.assert_frame_equal(positions, expected_positions)
|
||||
pd.testing.assert_frame_equal(fills, expected_fills)
|
||||
pd.testing.assert_frame_equal(pnl, expected_pnl)
|
||||
|
||||
|
||||
def test_generated_alpha_combo_portfolio_execution_workflow(tmp_path):
|
||||
daily_bars = make_generated_daily_bars()
|
||||
|
||||
computed_alpha = compute_alpha(
|
||||
data=daily_bars,
|
||||
alpha_name="generated_reversal_3d",
|
||||
alpha_type="reversal",
|
||||
lookback=3,
|
||||
)
|
||||
|
||||
assert list(computed_alpha.columns) == ALPHA_COLUMNS
|
||||
assert not computed_alpha.empty
|
||||
assert set(computed_alpha["symbol_id"]).issubset(set(GENERATED_SYMBOLS))
|
||||
assert computed_alpha["date"].min() > daily_bars["date"].min()
|
||||
assert computed_alpha["weight"].notna().all()
|
||||
assert computed_alpha["weight"].abs().sum() > 0.0
|
||||
assert {"ic", "rank_ic", "ir"}.isdisjoint(computed_alpha.columns)
|
||||
_assert_sorted_by_symbol_date(computed_alpha)
|
||||
|
||||
alpha_a = make_generated_alpha_weights("alpha_a", zero_date_index=2)
|
||||
alpha_b = make_generated_alpha_weights(
|
||||
"alpha_b",
|
||||
scale=0.5,
|
||||
offset=0.25,
|
||||
zero_date_index=2,
|
||||
)
|
||||
alpha_a_path = tmp_path / "alpha_a.pq"
|
||||
alpha_b_path = tmp_path / "alpha_b.pq"
|
||||
alpha_a.to_parquet(alpha_a_path, index=False)
|
||||
alpha_b.to_parquet(alpha_b_path, index=False)
|
||||
|
||||
identity_combo = combine_alphas([str(alpha_a_path)], "identity_combo")
|
||||
assert list(identity_combo.columns) == COMBO_COLUMNS
|
||||
assert (identity_combo["combo_name"] == "identity_combo").all()
|
||||
pd.testing.assert_frame_equal(
|
||||
identity_combo[["symbol_id", "date", "weight"]],
|
||||
alpha_a[["symbol_id", "date", "weight"]],
|
||||
)
|
||||
|
||||
equal_combo = combine_alphas([str(alpha_a_path), str(alpha_b_path)], "equal_combo")
|
||||
expected_equal_weights = (
|
||||
pd.concat([alpha_a, alpha_b], ignore_index=True)
|
||||
.groupby(["symbol_id", "date"], as_index=False)["weight"]
|
||||
.mean()
|
||||
.sort_values(["symbol_id", "date"])
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
pd.testing.assert_frame_equal(
|
||||
equal_combo[["symbol_id", "date", "weight"]],
|
||||
expected_equal_weights,
|
||||
)
|
||||
|
||||
portfolio_weights = make_generated_combo_weights("workflow_combo", zero_date_index=2)
|
||||
positions = construct_positions(
|
||||
weights_df=portfolio_weights,
|
||||
data_df=daily_bars,
|
||||
booksize=2_000_000.0,
|
||||
portfolio_name="workflow_portfolio",
|
||||
)
|
||||
|
||||
assert list(positions.columns) == POSITION_COLUMNS
|
||||
assert not positions.empty
|
||||
assert (positions["portfolio_name"] == "workflow_portfolio").all()
|
||||
assert pd.api.types.is_integer_dtype(positions["position_shares"])
|
||||
assert np.allclose(
|
||||
positions["position_value"],
|
||||
positions["position_shares"].astype(float) * positions["price"].fillna(0.0),
|
||||
)
|
||||
|
||||
target_gross_by_date = positions.groupby("date")["target_weight"].apply(lambda s: s.abs().sum())
|
||||
nonzero_target_dates = target_gross_by_date[target_gross_by_date > 0.0]
|
||||
assert np.allclose(nonzero_target_dates, 1.0)
|
||||
|
||||
nonzero_share_counts = positions.loc[positions["position_shares"] != 0, "position_shares"].abs()
|
||||
assert (nonzero_share_counts >= 100).all()
|
||||
|
||||
zero_gross_date = generated_sessions(10)[2]
|
||||
previous_date = generated_sessions(10)[1]
|
||||
zero_gross_positions = positions[positions["date"] == zero_gross_date].set_index("symbol_id")
|
||||
previous_positions = positions[positions["date"] == previous_date].set_index("symbol_id")
|
||||
common_symbols = zero_gross_positions.index.intersection(previous_positions.index)
|
||||
assert not common_symbols.empty
|
||||
assert (zero_gross_positions.loc[common_symbols, "target_weight"] == 0.0).all()
|
||||
pd.testing.assert_series_equal(
|
||||
zero_gross_positions.loc[common_symbols, "position_shares"],
|
||||
previous_positions.loc[common_symbols, "position_shares"],
|
||||
check_names=False,
|
||||
)
|
||||
|
||||
simulator = ReferenceSimulator(
|
||||
constraints=[
|
||||
SuspensionConstraint(),
|
||||
PriceLimitConstraint(),
|
||||
VolumeCapConstraint(max_frac=0.02),
|
||||
],
|
||||
cost_bps=5,
|
||||
slippage_bps=5,
|
||||
)
|
||||
fills, pnl = simulator.run(positions, daily_bars)
|
||||
|
||||
assert list(fills.columns) == FILL_COLUMNS
|
||||
assert list(pnl.columns) == PNL_COLUMNS
|
||||
assert not fills.empty
|
||||
assert not pnl.empty
|
||||
assert (fills["realized_shares"] == fills["prev_shares"] + fills["traded_shares"]).all()
|
||||
assert fills["blocked"].sum() > 0
|
||||
|
||||
fill_prices = fills.merge(
|
||||
daily_bars[["symbol_id", "date", "open"]],
|
||||
on=["symbol_id", "date"],
|
||||
how="left",
|
||||
validate="many_to_one",
|
||||
)
|
||||
expected_trade_cost = (
|
||||
fill_prices["traded_shares"].abs()
|
||||
* fill_prices["open"].fillna(0.0)
|
||||
* 10
|
||||
/ 10_000
|
||||
)
|
||||
assert np.allclose(fill_prices["trade_cost"], expected_trade_cost)
|
||||
|
||||
cost_by_date = fills.groupby("date")["trade_cost"].sum()
|
||||
assert np.allclose(
|
||||
pnl.set_index("date")["cost"],
|
||||
cost_by_date.reindex(pnl["date"], fill_value=0.0),
|
||||
)
|
||||
|
||||
booksize_used_by_simulator = positions.groupby("date")["target_value"].apply(lambda s: s.abs().sum()).max()
|
||||
traded_value_by_date = (
|
||||
fill_prices.assign(traded_value=fill_prices["traded_shares"].abs() * fill_prices["open"])
|
||||
.groupby("date")["traded_value"]
|
||||
.sum()
|
||||
)
|
||||
assert np.allclose(
|
||||
pnl.set_index("date")["turnover"],
|
||||
traded_value_by_date.reindex(pnl["date"], fill_value=0.0) / booksize_used_by_simulator,
|
||||
)
|
||||
|
||||
metrics = evaluate_portfolio(positions, daily_bars)
|
||||
_assert_metric_dict_is_finite(metrics)
|
||||
|
||||
|
||||
def test_generated_workflow_outputs_keep_parquet_schema_contracts(tmp_path):
|
||||
daily_bars = make_generated_daily_bars(n_sessions=10, include_missing=False)
|
||||
alpha = compute_alpha(
|
||||
data=daily_bars,
|
||||
alpha_name="schema_reversal",
|
||||
alpha_type="reversal",
|
||||
lookback=3,
|
||||
)
|
||||
alpha_path = tmp_path / "schema_reversal.pq"
|
||||
alpha.to_parquet(alpha_path, index=False)
|
||||
|
||||
combo = combine_alphas([str(alpha_path)], "schema_combo")
|
||||
positions = construct_positions(
|
||||
weights_df=combo,
|
||||
data_df=daily_bars,
|
||||
booksize=1_000_000.0,
|
||||
portfolio_name="schema_portfolio",
|
||||
)
|
||||
fills, pnl = ReferenceSimulator(cost_bps=5, slippage_bps=5).run(positions, daily_bars)
|
||||
|
||||
assert list(alpha.columns) == ALPHA_COLUMNS
|
||||
assert pd.api.types.is_object_dtype(alpha["symbol_id"])
|
||||
assert pd.api.types.is_datetime64_any_dtype(alpha["date"])
|
||||
assert pd.api.types.is_object_dtype(alpha["alpha_name"])
|
||||
assert pd.api.types.is_float_dtype(alpha["weight"])
|
||||
assert not alpha.isna().any().any()
|
||||
assert np.isfinite(alpha["weight"]).all()
|
||||
|
||||
assert list(combo.columns) == COMBO_COLUMNS
|
||||
assert pd.api.types.is_object_dtype(combo["symbol_id"])
|
||||
assert pd.api.types.is_datetime64_any_dtype(combo["date"])
|
||||
assert pd.api.types.is_object_dtype(combo["combo_name"])
|
||||
assert pd.api.types.is_float_dtype(combo["weight"])
|
||||
assert not combo.isna().any().any()
|
||||
assert np.isfinite(combo["weight"]).all()
|
||||
|
||||
assert list(positions.columns) == POSITION_COLUMNS
|
||||
assert pd.api.types.is_integer_dtype(positions["position_shares"])
|
||||
assert pd.api.types.is_datetime64_any_dtype(positions["date"])
|
||||
assert not positions.isna().any().any()
|
||||
position_numeric_columns = [
|
||||
"target_weight",
|
||||
"target_value",
|
||||
"target_shares",
|
||||
"position_value",
|
||||
"price",
|
||||
]
|
||||
assert np.isfinite(positions[position_numeric_columns]).all().all()
|
||||
|
||||
assert list(fills.columns) == FILL_COLUMNS
|
||||
assert pd.api.types.is_integer_dtype(fills["prev_shares"])
|
||||
assert pd.api.types.is_integer_dtype(fills["target_shares"])
|
||||
assert pd.api.types.is_integer_dtype(fills["traded_shares"])
|
||||
assert pd.api.types.is_integer_dtype(fills["realized_shares"])
|
||||
assert pd.api.types.is_integer_dtype(fills["blocked"])
|
||||
assert not fills.isna().any().any()
|
||||
assert np.isfinite(fills["trade_cost"]).all()
|
||||
|
||||
assert list(pnl.columns) == PNL_COLUMNS
|
||||
assert pd.api.types.is_integer_dtype(pnl["n_positions"])
|
||||
assert not pnl.isna().any().any()
|
||||
pnl_numeric_columns = [
|
||||
"gross_exposure",
|
||||
"net_exposure",
|
||||
"pnl",
|
||||
"cost",
|
||||
"turnover",
|
||||
]
|
||||
assert np.isfinite(pnl[pnl_numeric_columns]).all().all()
|
||||
|
||||
|
||||
def test_frozen_real_fixture_runs_high_level_workflow(tmp_path):
|
||||
real_daily_bars = pd.read_parquet(FIXTURE_PATH)
|
||||
|
||||
assert real_daily_bars.shape == (36, 19)
|
||||
assert set(real_daily_bars["symbol_id"]) == set(GENERATED_SYMBOLS)
|
||||
assert real_daily_bars["date"].min() == pd.Timestamp("2024-01-02")
|
||||
assert real_daily_bars["date"].max() == pd.Timestamp("2024-01-12")
|
||||
assert real_daily_bars.groupby("date")["symbol_id"].nunique().eq(4).all()
|
||||
|
||||
reversal_alpha = compute_alpha(
|
||||
data=real_daily_bars,
|
||||
alpha_name="real_reversal_3d",
|
||||
alpha_type="reversal",
|
||||
lookback=3,
|
||||
)
|
||||
reversal_vol_alpha = compute_alpha(
|
||||
data=real_daily_bars,
|
||||
alpha_name="real_reversal_vol_3d",
|
||||
alpha_type="reversal_vol",
|
||||
lookback=3,
|
||||
vol_window=3,
|
||||
)
|
||||
|
||||
reversal_path = tmp_path / "real_reversal.pq"
|
||||
reversal_vol_path = tmp_path / "real_reversal_vol.pq"
|
||||
reversal_alpha.to_parquet(reversal_path, index=False)
|
||||
reversal_vol_alpha.to_parquet(reversal_vol_path, index=False)
|
||||
|
||||
combo = combine_alphas([str(reversal_path), str(reversal_vol_path)], "real_equal_combo")
|
||||
positions = construct_positions(
|
||||
weights_df=combo,
|
||||
data_df=real_daily_bars,
|
||||
booksize=1_000_000.0,
|
||||
portfolio_name="real_fixture_portfolio",
|
||||
)
|
||||
fills, pnl = ReferenceSimulator(cost_bps=5, slippage_bps=5).run(positions, real_daily_bars)
|
||||
metrics = evaluate_portfolio(positions, real_daily_bars)
|
||||
|
||||
assert not reversal_alpha.empty
|
||||
assert not reversal_vol_alpha.empty
|
||||
assert not combo.empty
|
||||
assert not positions.empty
|
||||
assert not fills.empty
|
||||
assert not pnl.empty
|
||||
assert np.isfinite(combo["weight"]).all()
|
||||
assert np.isfinite(positions["target_weight"]).all()
|
||||
assert np.isfinite(pnl[["gross_exposure", "net_exposure", "pnl", "cost", "turnover"]]).all().all()
|
||||
_assert_metric_dict_is_finite(metrics)
|
||||
Reference in New Issue
Block a user