Raise coverage threshold to 95% and expand test coverage

- pyproject.toml: fail_under 80 → 95
- test_alpha: +79 lines
- test_cli_workflow: +226 lines
- test_derived: +121 lines
- test_downloader_contracts: +169 lines
- test_features: +16 lines
- test_minute_downloader: +81 lines
- test_portfolio: +208 lines
This commit is contained in:
Yuxuan Yan
2026-06-16 21:10:30 +08:00
parent b5c8c0b8da
commit 528620b271
8 changed files with 898 additions and 4 deletions
+1 -1
View File
@@ -39,7 +39,7 @@ source = [
] ]
[tool.coverage.report] [tool.coverage.report]
fail_under = 80 fail_under = 95
show_missing = true show_missing = true
skip_covered = false skip_covered = false
omit = [ omit = [
+79
View File
@@ -191,6 +191,15 @@ def test_combine_single_alpha_is_identity(tmp_path):
assert (combo["combo_name"] == "rev_combo").all() assert (combo["combo_name"] == "rev_combo").all()
def test_combine_alphas_rejects_unknown_method(tmp_path):
data = _make_data()
alpha_path = tmp_path / "alpha.pq"
compute_alpha(data, "rev", "reversal", lookback=5).to_parquet(alpha_path, index=False)
with pytest.raises(ValueError, match="Unknown combo method"):
combine_alphas([str(alpha_path)], "bad_combo", method="does_not_exist")
# --- registry / factory ----------------------------------------------------- # --- registry / factory -----------------------------------------------------
def test_builtins_are_registered(): def test_builtins_are_registered():
@@ -205,6 +214,22 @@ def test_get_alpha_filters_unaccepted_params():
assert not hasattr(alpha, "vol_window") assert not hasattr(alpha, "vol_window")
def test_get_alpha_forwards_kwargs_to_flexible_alpha():
@register_alpha
class _FlexibleAlpha(BaseAlpha):
name = "_flexible_alpha_kwargs"
def __init__(self, **kwargs):
self.kwargs = kwargs
def signal(self, close):
return close
alpha = get_alpha("_flexible_alpha_kwargs", decay=0.5, label="demo")
assert alpha.kwargs == {"decay": 0.5, "label": "demo"}
def test_get_alpha_unknown_raises(): def test_get_alpha_unknown_raises():
with pytest.raises(KeyError): with pytest.raises(KeyError):
get_alpha("does_not_exist") get_alpha("does_not_exist")
@@ -227,6 +252,31 @@ def test_register_rejects_non_basealpha():
register_alpha(object) # type: ignore[arg-type] register_alpha(object) # type: ignore[arg-type]
def test_register_rejects_empty_alpha_name():
with pytest.raises(ValueError, match="non-empty"):
@register_alpha
class NoNameAlpha(BaseAlpha):
def signal(self, close):
return close
def test_load_alpha_module_error_paths(tmp_path, monkeypatch):
missing_path = tmp_path / "missing_alpha.py"
with pytest.raises(FileNotFoundError):
load_alpha_module(str(missing_path))
bad_path = tmp_path / "bad_alpha.py"
bad_path.write_text("x = 1\n")
monkeypatch.setattr(
"pipeline.alpha.registry.importlib.util.spec_from_file_location",
lambda *args, **kwargs: None,
)
with pytest.raises(ImportError, match="Cannot load alpha module"):
load_alpha_module(str(bad_path))
load_alpha_module("math")
# --- base class -------------------------------------------------------------- # --- base class --------------------------------------------------------------
def test_to_weights_are_per_date_zscore(): def test_to_weights_are_per_date_zscore():
@@ -242,6 +292,20 @@ def test_to_weights_are_per_date_zscore():
assert (weights.mean(axis=1).abs() < 1e-9).all() assert (weights.mean(axis=1).abs() < 1e-9).all()
def test_base_alpha_default_signal_and_repr():
alpha = BaseAlpha()
alpha.example = 3
with pytest.raises(NotImplementedError, match="signal"):
alpha.signal(pd.DataFrame({"x": [1.0]}))
with pytest.raises(NotImplementedError, match="signal"):
alpha.signal_from_data(
pd.DataFrame({"symbol_id": ["sh600000"]}),
pd.DataFrame({"sh600000": [1.0]}),
)
assert repr(alpha) == "BaseAlpha(example=3)"
# --- external plugin loading ------------------------------------------------- # --- external plugin loading -------------------------------------------------
def test_load_external_alpha_module(tmp_path): def test_load_external_alpha_module(tmp_path):
@@ -502,3 +566,18 @@ def test_compute_alpha_rejects_feature_path_collision_with_daily_data(tmp_path):
"reversal", "reversal",
feature_paths=[str(close_collision_path)], feature_paths=[str(close_collision_path)],
) )
def test_evaluate_alpha_empty_when_signal_dates_not_on_market_calendar():
data = _make_data(n_days=3)
alpha = pd.DataFrame({
"symbol_id": ["sh600000"],
"date": [pd.Timestamp("2030-01-01")],
"alpha_name": ["future"],
"weight": [1.0],
})
metrics = evaluate_alpha(alpha, data)
assert metrics["n_dates"] == 0
assert metrics["cumulative_return"] == 0.0
+226
View File
@@ -5,10 +5,13 @@ from __future__ import annotations
import textwrap import textwrap
from pathlib import Path from pathlib import Path
import click
import pandas as pd import pandas as pd
from click.testing import CliRunner from click.testing import CliRunner
from cli import cli from cli import cli
import pipeline.derived.cli as derived_cli
import pipeline.features.cli as features_cli
from tests.helpers import ( from tests.helpers import (
make_generated_daily_bars, make_generated_daily_bars,
make_generated_derived_features, make_generated_derived_features,
@@ -426,6 +429,61 @@ def test_cli_error_paths_are_clear_for_bad_user_inputs(tmp_path):
]) ])
assert "Symbol 'sh999999' not found" in alphaview_missing_symbol.output assert "Symbol 'sh999999' not found" in alphaview_missing_symbol.output
alphaview_missing_column = _invoke_error(runner, [
"alphaview",
"--data-path", str(daily_path),
"--alpha-path", str(positions_path),
"--symbol", "sh600000",
"--columns", "close,missing_bar_col",
])
assert "Bar columns not found: missing_bar_col" in alphaview_missing_column.output
alphaview_alpha = pd.DataFrame({
"symbol_id": ["sh600000"],
"date": [daily_bars["date"].min()],
"alpha_name": ["toy_alpha"],
"weight": [1.0],
})
alphaview_alpha_path = tmp_path / "alphaview_alpha.pq"
alphaview_alpha.to_parquet(alphaview_alpha_path, index=False)
alphaview_empty_range = _invoke_error(runner, [
"alphaview",
"--data-path", str(daily_path),
"--alpha-path", str(alphaview_alpha_path),
"--symbol", "sh600000",
"--start-date", "2030-01-01",
])
assert "No rows in the requested date range" in alphaview_empty_range.output
empty_combo_paths = _invoke_ok(runner, [
"combo", "combine",
"--alpha-paths", " , ",
"--combo-name", "empty",
"--output-dir", str(tmp_path / "combos"),
])
assert "requires at least 1 path" in empty_combo_paths.output
def test_cli_parser_helpers_cover_string_coercion_and_bad_params():
assert derived_cli._parse_params(("n=7", "scale=2.5", "label=demo")) == {
"n": 7,
"scale": 2.5,
"label": "demo",
}
assert features_cli._parse_params(("n=7", "scale=2.5", "label=demo")) == {
"n": 7,
"scale": 2.5,
"label": "demo",
}
for module in (derived_cli, features_cli):
try:
module._parse_params(("not-an-assignment",))
except click.BadParameter as exc:
assert "--param must be name=value" in str(exc)
else:
raise AssertionError("expected BadParameter")
def test_cli_list_and_legacy_feature_paths(tmp_path): def test_cli_list_and_legacy_feature_paths(tmp_path):
runner = CliRunner() runner = CliRunner()
@@ -519,6 +577,174 @@ def test_cli_list_and_legacy_feature_paths(tmp_path):
assert "Unknown feature-type" in unknown_feature.output assert "Unknown feature-type" in unknown_feature.output
def test_cli_shortcuts_and_external_module_loading(tmp_path):
runner = CliRunner()
daily_bars = make_generated_daily_bars(n_sessions=8, include_missing=False)
minute_bars = make_generated_minute_bars(daily_bars)
daily_path = tmp_path / "daily_bars.pq"
minute_path = tmp_path / "minute_bars.pq"
daily_bars.to_parquet(daily_path, index=False)
minute_bars.to_parquet(minute_path, index=False)
alpha_module = tmp_path / "listed_alpha.py"
alpha_module.write_text(textwrap.dedent("""
import pandas as pd
from pipeline.alpha.base import BaseAlpha
from pipeline.alpha.registry import register_alpha
@register_alpha
class ListedAlpha(BaseAlpha):
name = "listed_alpha_cli"
def __init__(self, scale: float = 1.0, label: str = "x"):
self.scale = scale
self.label = label
def signal(self, close: pd.DataFrame) -> pd.DataFrame:
return close.pct_change(1, fill_method=None) * self.scale
"""))
derived_module = tmp_path / "listed_derived.py"
derived_module.write_text(textwrap.dedent("""
import pandas as pd
from pipeline.derived.base import BaseDerivedData
from pipeline.derived.registry import register_derived
@register_derived
class ListedDerived(BaseDerivedData):
name = "listed_derived_cli"
def __init__(self, scale: float = 1.0):
self.scale = scale
def compute(self, daily=None, minute=None) -> pd.DataFrame:
out = daily[["symbol_id", "date", "close"]].copy()
out["listed_value"] = out.pop("close") * self.scale
return out
"""))
derived_compute_module = tmp_path / "computed_derived.py"
derived_compute_module.write_text(textwrap.dedent("""
import pandas as pd
from pipeline.derived.base import BaseDerivedData
from pipeline.derived.registry import register_derived
@register_derived
class ComputedDerived(BaseDerivedData):
name = "computed_derived_cli"
def __init__(self, scale: float = 1.0):
self.scale = scale
def compute(self, daily=None, minute=None) -> pd.DataFrame:
out = daily[["symbol_id", "date", "close"]].copy()
out["computed_value"] = out.pop("close") * self.scale
return out
"""))
feature_module = tmp_path / "listed_feature.py"
feature_module.write_text(textwrap.dedent("""
import pandas as pd
from pipeline.features.base import BaseFeature
from pipeline.features.registry import register_feature
@register_feature
class ListedFeature(BaseFeature):
name = "listed_feature_cli"
def compute(self, daily=None, minute=None) -> pd.DataFrame:
out = minute[["symbol_id", "date", "close"]].copy()
out["date"] = pd.to_datetime(out["date"]).dt.normalize()
out = out.groupby(["symbol_id", "date"], as_index=False)["close"].mean()
return out.rename(columns={"close": "listed_feature_value"})
"""))
feature_compute_module = tmp_path / "computed_feature.py"
feature_compute_module.write_text(textwrap.dedent("""
import pandas as pd
from pipeline.features.base import BaseFeature
from pipeline.features.registry import register_feature
@register_feature
class ComputedFeature(BaseFeature):
name = "computed_feature_cli"
def compute(self, daily=None, minute=None) -> pd.DataFrame:
out = minute[["symbol_id", "date", "close"]].copy()
out["date"] = pd.to_datetime(out["date"]).dt.normalize()
out = out.groupby(["symbol_id", "date"], as_index=False)["close"].mean()
return out.rename(columns={"close": "computed_feature_value"})
"""))
alpha_list = _invoke_ok(runner, [
"alpha", "list",
"--alpha-module", str(alpha_module),
])
assert "listed_alpha_cli" in alpha_list.output
alpha_dir = tmp_path / "alphas"
reversal = _invoke_ok(runner, [
"alpha", "reversal",
"--data-path", str(daily_path),
"--output-dir", str(alpha_dir),
"--lookback", "3",
])
reversal_vol = _invoke_ok(runner, [
"alpha", "reversal-vol",
"--data-path", str(daily_path),
"--output-dir", str(alpha_dir),
"--lookback", "3",
"--vol-window", "3",
])
external_alpha = _invoke_ok(runner, [
"alpha", "compute",
"--data-path", str(daily_path),
"--alpha-type", "listed_alpha_cli",
"--alpha-name", "listed_alpha_run",
"--param", "scale=2.5",
"--param", "label=demo",
"--output-dir", str(alpha_dir),
])
assert "Saved alpha:" in reversal.output
assert "Saved alpha:" in reversal_vol.output
assert "Saved alpha:" in external_alpha.output
assert (alpha_dir / "reversal_3d.pq").exists()
assert (alpha_dir / "reversal_vol_3d_3d.pq").exists()
assert (alpha_dir / "listed_alpha_run.pq").exists()
derived_list = _invoke_ok(runner, [
"derived", "list",
"--derived-module", str(derived_module),
])
assert "listed_derived_cli" in derived_list.output
derived_dir = tmp_path / "derived_external"
derived_compute = _invoke_ok(runner, [
"derived", "compute",
"--daily-path", str(daily_path),
"--derived-module", str(derived_compute_module),
"--derived-type", "computed_derived_cli",
"--derived-name", "listed_derived_run",
"--param", "scale=3",
"--output-dir", str(derived_dir),
])
assert "Saved derived data:" in derived_compute.output
assert (derived_dir / "listed_derived_run.pq").exists()
feature_list = _invoke_ok(runner, [
"feature", "list",
"--feature-module", str(feature_module),
])
assert "listed_feature_cli" in feature_list.output
feature_dir = tmp_path / "features_external"
feature_compute = _invoke_ok(runner, [
"feature", "compute",
"--minute-path", str(minute_path),
"--feature-module", str(feature_compute_module),
"--feature-type", "computed_feature_cli",
"--feature-name", "listed_feature_run",
"--output-dir", str(feature_dir),
])
assert "Saved feature:" in feature_compute.output
assert (feature_dir / "listed_feature_run.pq").exists()
def test_cli_pqcat_row_modes(tmp_path): def test_cli_pqcat_row_modes(tmp_path):
runner = CliRunner() runner = CliRunner()
daily_bars = make_generated_daily_bars(n_sessions=3, include_missing=False) daily_bars = make_generated_daily_bars(n_sessions=3, include_missing=False)
+118 -3
View File
@@ -9,8 +9,14 @@ from click.testing import CliRunner
from cli import cli from cli import cli
from pipeline.alpha.compute import join_feature_frames from pipeline.alpha.compute import join_feature_frames
from pipeline.derived.compute import compute_derived, validate_derived_frame from pipeline.derived.base import BaseDerivedData
from pipeline.derived.registry import available_derived, get_derived, load_derived_module from pipeline.derived.compute import compute_derived, read_derived_frame, validate_derived_frame
from pipeline.derived.registry import (
available_derived,
get_derived,
load_derived_module,
register_derived,
)
def _daily_bars() -> pd.DataFrame: def _daily_bars() -> pd.DataFrame:
@@ -88,6 +94,27 @@ def test_validate_derived_frame_rejects_non_numeric_values():
})) }))
def test_validate_derived_frame_rejects_missing_value_columns():
with pytest.raises(ValueError, match="at least one value column"):
validate_derived_frame(pd.DataFrame({
"symbol_id": ["sh600000"],
"date": [pd.Timestamp("2024-01-02")],
}))
def test_read_derived_frame_rejects_empty_csv(tmp_path):
empty_csv = tmp_path / "empty.csv"
empty_csv.write_text("")
with pytest.raises(ValueError, match="CSV input is empty"):
read_derived_frame(empty_csv)
def test_compute_derived_rejects_missing_inputs():
with pytest.raises(ValueError, match="requires --daily-path or --minute-path"):
compute_derived("minute_daily_summary")
def test_derived_ingest_cli_accepts_csv_and_parquet(tmp_path): def test_derived_ingest_cli_accepts_csv_and_parquet(tmp_path):
runner = CliRunner() runner = CliRunner()
source = pd.DataFrame({ source = pd.DataFrame({
@@ -136,6 +163,26 @@ def test_derived_validate_cli_rejects_duplicate_csv_columns(tmp_path):
assert "duplicate columns" in result.output assert "duplicate columns" in result.output
def test_derived_ingest_cli_wraps_validation_errors(tmp_path):
runner = CliRunner()
csv_path = tmp_path / "bad_ingest.csv"
csv_path.write_text("symbol_id,date\nsh600000,2024-01-02\n")
result = runner.invoke(cli, [
"derived",
"ingest",
"--input-path",
str(csv_path),
"--derived-name",
"bad",
"--output-dir",
str(tmp_path / "derived"),
])
assert result.exit_code != 0
assert "at least one value column" in result.output
def test_external_derived_plugin_loads_filters_params_and_uses_inputs(tmp_path): def test_external_derived_plugin_loads_filters_params_and_uses_inputs(tmp_path):
module_path = tmp_path / "external_derived.py" module_path = tmp_path / "external_derived.py"
module_path.write_text(textwrap.dedent(''' module_path.write_text(textwrap.dedent('''
@@ -204,6 +251,57 @@ def test_external_derived_plugin_loads_filters_params_and_uses_inputs(tmp_path):
assert {"daily_scaled_close", "minute_volume_sum"}.issubset(both_result.columns) assert {"daily_scaled_close", "minute_volume_sum"}.issubset(both_result.columns)
def test_derived_registry_rejects_bad_plugins_and_load_failures(tmp_path, monkeypatch):
with pytest.raises(TypeError):
register_derived(object) # type: ignore[arg-type]
with pytest.raises(ValueError, match="non-empty"):
@register_derived
class NoNameDerived(BaseDerivedData):
def compute(self, daily=None, minute=None):
return pd.DataFrame()
@register_derived
class _CoverageDerived(BaseDerivedData):
name = "_coverage_derived_registry"
def compute(self, daily=None, minute=None):
return pd.DataFrame({
"symbol_id": ["sh600000"],
"date": [pd.Timestamp("2024-01-02")],
"x": [1.0],
})
with pytest.raises(ValueError, match="already registered"):
@register_derived
class _CoverageDerivedDuplicate(BaseDerivedData):
name = "_coverage_derived_registry"
def compute(self, daily=None, minute=None):
return pd.DataFrame()
with pytest.raises(KeyError, match="Unknown derived data"):
get_derived("does_not_exist")
missing_path = tmp_path / "missing_derived.py"
with pytest.raises(FileNotFoundError):
load_derived_module(str(missing_path))
bad_path = tmp_path / "bad_derived.py"
bad_path.write_text("x = 1\n")
monkeypatch.setattr(
"pipeline.derived.registry.importlib.util.spec_from_file_location",
lambda *args, **kwargs: None,
)
with pytest.raises(ImportError, match="Cannot load derived data module"):
load_derived_module(str(bad_path))
load_derived_module("math")
instance = _CoverageDerived()
instance.scale = 2
assert repr(instance) == "_CoverageDerived(scale=2)"
def test_derived_compute_cli_writes_builtin_minute_summary(tmp_path): def test_derived_compute_cli_writes_builtin_minute_summary(tmp_path):
runner = CliRunner() runner = CliRunner()
minute_path = tmp_path / "minute.pq" minute_path = tmp_path / "minute.pq"
@@ -223,6 +321,24 @@ def test_derived_compute_cli_writes_builtin_minute_summary(tmp_path):
assert "minute_vwap" in written.columns assert "minute_vwap" in written.columns
def test_minute_summary_uses_time_sort_and_daily_without_close():
minute = _minute_bars().drop(columns=["datetime"]).sample(frac=1.0, random_state=0)
daily = _daily_bars()[["symbol_id", "date", "open"]]
result = compute_derived(
"minute_daily_summary",
daily=daily,
minute=minute,
)
by_symbol = result.set_index(["symbol_id", "date"])
assert np.isclose(
by_symbol.loc[("sh600000", pd.Timestamp("2024-01-02")), "minute_intraday_return"],
0.10,
)
assert "minute_vwap_deviation" in result.columns
def test_alpha_feature_join_rejects_derived_column_collisions(): def test_alpha_feature_join_rejects_derived_column_collisions():
data = _daily_bars() data = _daily_bars()
derived_a = data[["symbol_id", "date"]].copy() derived_a = data[["symbol_id", "date"]].copy()
@@ -258,4 +374,3 @@ def test_legacy_feature_cli_delegates_to_derived_registry(tmp_path):
]) ])
assert compute_result.exit_code == 0, compute_result.output assert compute_result.exit_code == 0, compute_result.output
assert (out_dir / "minute_summary.pq").exists() assert (out_dir / "minute_summary.pq").exists()
+169
View File
@@ -132,6 +132,41 @@ def test_download_daily_raises_when_requested_source_has_no_data(monkeypatch):
) )
def test_download_daily_akshare_source_skips_baostock(monkeypatch):
calls: list[str] = []
fallback = pd.DataFrame({
"symbol": ["sh600000"],
"date": ["2024-01-02"],
"open": [10.0],
"high": [11.0],
"low": [9.0],
"close": [10.5],
"volume": [1000.0],
"amount": [10500.0],
})
monkeypatch.setattr(
downloader,
"_download_baostock",
lambda *args: calls.append("baostock") or None,
)
monkeypatch.setattr(
downloader,
"_download_akshare",
lambda *args: calls.append("akshare") or fallback,
)
result = download_daily(
"sh600000",
"2024-01-02",
"2024-01-02",
source="akshare",
)
assert calls == ["akshare"]
assert result["date"].tolist() == [pd.Timestamp("2024-01-02")]
def test_akshare_daily_downloader_maps_columns_and_failures(monkeypatch): def test_akshare_daily_downloader_maps_columns_and_failures(monkeypatch):
calls: list[dict] = [] calls: list[dict] = []
raw = pd.DataFrame({ raw = pd.DataFrame({
@@ -326,6 +361,34 @@ def test_download_daily_batch_periodic_relogin_and_none_result(monkeypatch):
assert logout_count == 2 assert logout_count == 2
def test_download_daily_batch_empty_rows_yields_none(monkeypatch):
monkeypatch.setattr(downloader.bs, "login", lambda: None)
monkeypatch.setattr(downloader.bs, "logout", lambda: None)
monkeypatch.setattr(
downloader.bs,
"query_history_k_data_plus",
lambda **kwargs: _FakeResult([]),
)
assert list(download_daily_batch(["sh600000"], "2024-01-02", "2024-01-02")) == [
("sh600000", None)
]
def test_download_daily_batch_generic_exception_yields_none(monkeypatch):
monkeypatch.setattr(downloader.bs, "login", lambda: None)
monkeypatch.setattr(downloader.bs, "logout", lambda: None)
monkeypatch.setattr(
downloader.bs,
"query_history_k_data_plus",
lambda **kwargs: (_ for _ in ()).throw(RuntimeError("query failed")),
)
assert list(download_daily_batch(["sh600000"], "2024-01-02", "2024-01-02")) == [
("sh600000", None)
]
def test_download_daily_batch_relogs_and_retries_session_loss(monkeypatch): def test_download_daily_batch_relogs_and_retries_session_loss(monkeypatch):
responses = [ responses = [
_FakeResult([], error_code="10002007", error_msg="用户未登录"), _FakeResult([], error_code="10002007", error_msg="用户未登录"),
@@ -359,6 +422,32 @@ def test_download_daily_batch_relogs_and_retries_session_loss(monkeypatch):
assert logout_count == 2 assert logout_count == 2
def test_download_daily_batch_second_session_loss_and_logout_failure(monkeypatch):
responses = [
_FakeResult([], error_code="10002007", error_msg="用户未登录"),
_FakeResult([], error_code="10002007", error_msg="用户未登录"),
]
logout_count = 0
def fake_logout():
nonlocal logout_count
logout_count += 1
raise RuntimeError("logout failed")
monkeypatch.setattr(downloader.bs, "login", lambda: None)
monkeypatch.setattr(downloader.bs, "logout", fake_logout)
monkeypatch.setattr(
downloader.bs,
"query_history_k_data_plus",
lambda **kwargs: responses.pop(0),
)
assert list(download_daily_batch(["sh600000"], "2024-01-02", "2024-01-02")) == [
("sh600000", None)
]
assert logout_count == 2
def test_download_daily_batch_uses_akshare_fallback_when_enabled(monkeypatch): def test_download_daily_batch_uses_akshare_fallback_when_enabled(monkeypatch):
fallback = pd.DataFrame({ fallback = pd.DataFrame({
"symbol": ["sh600000"], "symbol": ["sh600000"],
@@ -438,6 +527,10 @@ def test_download_universe_writes_daily_partitions_from_mock_batch(tmp_path, mon
monkeypatch.setattr(pipeline_downloader, "download_daily_batch", fake_batch) monkeypatch.setattr(pipeline_downloader, "download_daily_batch", fake_batch)
stale_file = tmp_path / "toy" / "month=2024-01" / "stale.pq"
stale_file.parent.mkdir(parents=True)
batch_frame.iloc[[0]][DATA_COLUMNS[2:]].to_parquet(stale_file, index=False)
stats = download_universe( stats = download_universe(
universe="toy", universe="toy",
start_date="2024-01-02", start_date="2024-01-02",
@@ -458,5 +551,81 @@ def test_download_universe_writes_daily_partitions_from_mock_batch(tmp_path, mon
} }
assert (dataset_path / "month=2024-01").exists() assert (dataset_path / "month=2024-01").exists()
assert (dataset_path / "month=2024-02").exists() assert (dataset_path / "month=2024-02").exists()
assert not stale_file.exists()
assert written[DATA_COLUMNS].columns.tolist() == DATA_COLUMNS assert written[DATA_COLUMNS].columns.tolist() == DATA_COLUMNS
assert written["symbol_name"].tolist() == ["PF Bank", "PF Bank"] assert written["symbol_name"].tolist() == ["PF Bank", "PF Bank"]
def test_download_universe_raises_when_all_daily_symbols_empty(tmp_path, monkeypatch):
monkeypatch.setattr(
pipeline_downloader,
"_resolve_universe",
lambda universe, max_symbols=0: pd.DataFrame({
"symbol_id": ["sh600000"],
"symbol_name": ["PF Bank"],
}),
)
monkeypatch.setattr(
pipeline_downloader,
"download_daily_batch",
lambda symbols, start, end, adjust="qfq": iter([("sh600000", None)]),
)
with pytest.raises(RuntimeError, match="No data downloaded"):
download_universe(
universe="toy",
start_date="2024-01-02",
end_date="2024-01-02",
output_dir=str(tmp_path),
)
def test_download_universe_progress_branch_at_100_symbols(tmp_path, monkeypatch):
symbols = [f"sh6{i:05d}" for i in range(100)]
batch_frame = pd.DataFrame({
"symbol": ["sh600000"],
"date": [pd.Timestamp("2024-01-02")],
"open": [10.0],
"high": [11.0],
"low": [9.0],
"close": [10.5],
"preclose": [10.0],
"volume": [1000.0],
"amount": [10500.0],
"vwap": [10.5],
"turn": [1.0],
"pctChg": [5.0],
"tradestatus": [1],
"isST": [0],
"peTTM": [8.0],
"pbMRQ": [1.1],
"psTTM": [2.1],
"pcfNcfTTM": [3.1],
})
monkeypatch.setattr(
pipeline_downloader,
"_resolve_universe",
lambda universe, max_symbols=0: pd.DataFrame({
"symbol_id": symbols,
"symbol_name": symbols,
}),
)
def fake_batch(requested_symbols, start, end, adjust="qfq"):
assert requested_symbols == symbols
for symbol in requested_symbols:
yield symbol, batch_frame.copy()
monkeypatch.setattr(pipeline_downloader, "download_daily_batch", fake_batch)
stats = download_universe(
universe="toy100",
start_date="2024-01-02",
end_date="2024-01-02",
output_dir=str(tmp_path),
chunk_size=200,
)
assert stats["n_symbols"] == 100
assert stats["n_rows"] == 100
+16
View File
@@ -7,6 +7,7 @@ import pandas as pd
import pytest import pytest
from pipeline.features.compute import compute_feature, validate_feature_frame from pipeline.features.compute import compute_feature, validate_feature_frame
from pipeline.features.compute import read_feature_frames
from pipeline.features.library.minute_daily_summary import MinuteDailySummaryFeature from pipeline.features.library.minute_daily_summary import MinuteDailySummaryFeature
from pipeline.features.registry import ( from pipeline.features.registry import (
available_features, available_features,
@@ -99,6 +100,21 @@ def test_legacy_feature_compute_matches_canonical_derived_compute():
pd.testing.assert_frame_equal(legacy_feature, canonical_derived) pd.testing.assert_frame_equal(legacy_feature, canonical_derived)
def test_read_feature_frames_delegates_to_derived_validation(tmp_path):
feature = pd.DataFrame({
"symbol_id": ["sh600000"],
"date": ["2024-01-02 15:00:00"],
"toy_feature": [1.5],
})
feature_path = tmp_path / "feature.pq"
feature.to_parquet(feature_path, index=False)
[result] = read_feature_frames([feature_path])
assert result["date"].tolist() == [pd.Timestamp("2024-01-02")]
assert result["toy_feature"].tolist() == [1.5]
def test_load_external_feature_module_and_filter_params(tmp_path): def test_load_external_feature_module_and_filter_params(tmp_path):
module_path = tmp_path / "external_feature.py" module_path = tmp_path / "external_feature.py"
module_path.write_text(textwrap.dedent(''' module_path.write_text(textwrap.dedent('''
+81
View File
@@ -178,6 +178,37 @@ def test_download_minute_batch_second_session_loss_yields_none(monkeypatch):
] ]
def test_download_minute_batch_ignores_relogin_and_final_logout_failures(monkeypatch):
responses = [
_FakeResult([], error_code="1", error_msg="bad symbol"),
_FakeResult([]),
]
logout_count = 0
def fake_logout():
nonlocal logout_count
logout_count += 1
raise RuntimeError("logout failed")
monkeypatch.setattr(low_level_downloader.bs, "login", lambda: None)
monkeypatch.setattr(low_level_downloader.bs, "logout", fake_logout)
monkeypatch.setattr(
low_level_downloader.bs,
"query_history_k_data_plus",
lambda **kwargs: responses.pop(0),
)
assert list(
download_minute_batch(
["sh600000", "sz000001"],
"2024-01-02",
"2024-01-02",
relogin_every=1,
)
) == [("sh600000", None), ("sz000001", None)]
assert logout_count == 2
def test_download_minute_batch_rejects_unparsed_timestamps(monkeypatch): def test_download_minute_batch_rejects_unparsed_timestamps(monkeypatch):
bad_rows = [[ bad_rows = [[
"2024-01-02", "2024-01-02",
@@ -244,6 +275,9 @@ def test_download_minute_universe_writes_frequency_month_partitions(tmp_path, mo
preserved_minute["symbol_id"] = "sh600000" preserved_minute["symbol_id"] = "sh600000"
preserved_minute["symbol_name"] = "PF Bank" preserved_minute["symbol_name"] = "PF Bank"
preserved_minute[MINUTE_BAR_COLUMNS].to_parquet(preserved, index=False) preserved_minute[MINUTE_BAR_COLUMNS].to_parquet(preserved, index=False)
stale = tmp_path / "toy" / "frequency=5m" / "month=2024-01" / "stale.pq"
stale.parent.mkdir(parents=True)
preserved_minute.assign(frequency="5m")[MINUTE_BAR_COLUMNS].to_parquet(stale, index=False)
stats = download_minute_universe( stats = download_minute_universe(
universe="toy", universe="toy",
@@ -257,6 +291,7 @@ def test_download_minute_universe_writes_frequency_month_partitions(tmp_path, mo
dataset_path = Path(stats["dataset_path"]) dataset_path = Path(stats["dataset_path"])
assert (dataset_path / "frequency=5m" / "month=2024-01").is_dir() assert (dataset_path / "frequency=5m" / "month=2024-01").is_dir()
assert preserved.exists() assert preserved.exists()
assert not stale.exists()
out = pd.read_parquet(dataset_path / "frequency=5m") out = pd.read_parquet(dataset_path / "frequency=5m")
assert (set(MINUTE_BAR_COLUMNS) - {"frequency"}) <= set(out.columns) assert (set(MINUTE_BAR_COLUMNS) - {"frequency"}) <= set(out.columns)
assert set(out["symbol_id"]) == {"sh600000"} assert set(out["symbol_id"]) == {"sh600000"}
@@ -286,3 +321,49 @@ def test_download_minute_universe_raises_when_all_symbols_empty(tmp_path, monkey
end_date="2024-01-02", end_date="2024-01-02",
output_dir=str(tmp_path), output_dir=str(tmp_path),
) )
def test_download_minute_universe_progress_branch_at_100_symbols(tmp_path, monkeypatch):
symbols = [f"sh6{i:05d}" for i in range(100)]
minute = pd.DataFrame({
"symbol": ["sh600000"],
"datetime": [pd.Timestamp("2024-01-02 09:35:00")],
"date": [pd.Timestamp("2024-01-02")],
"time": ["09:35:00"],
"frequency": ["5m"],
"open": [10.0],
"high": [11.0],
"low": [9.0],
"close": [10.5],
"volume": [1000.0],
"amount": [10500.0],
"vwap": [10.5],
"adjustflag": ["3"],
})
monkeypatch.setattr(
pipeline_downloader,
"_resolve_universe",
lambda universe, max_symbols=0: pd.DataFrame({
"symbol_id": symbols,
"symbol_name": symbols,
}),
)
def fake_batch(requested_symbols, start, end, frequency=5):
assert requested_symbols == symbols
for symbol in requested_symbols:
yield symbol, minute.copy()
monkeypatch.setattr(pipeline_downloader, "download_minute_batch", fake_batch)
stats = download_minute_universe(
universe="toy100",
start_date="2024-01-02",
end_date="2024-01-02",
output_dir=str(tmp_path),
chunk_size=200,
)
assert stats["n_symbols"] == 100
assert stats["n_rows"] == 100
+208
View File
@@ -13,12 +13,17 @@ from pipeline.portfolio.market_rules import (
Board, Board,
LimitStatus, LimitStatus,
MarketRule, MarketRule,
_to_date,
compute_limit_status, compute_limit_status,
detect_board, detect_board,
) )
from pipeline.portfolio.research import evaluate_portfolio from pipeline.portfolio.research import evaluate_portfolio
from pipeline.portfolio.constraints import ( from pipeline.portfolio.constraints import (
TradeConstraint,
available_constraints,
get_constraint,
PriceLimitConstraint, PriceLimitConstraint,
register_constraint,
SuspensionConstraint, SuspensionConstraint,
VolumeCapConstraint, VolumeCapConstraint,
) )
@@ -82,6 +87,7 @@ def test_detect_board():
assert detect_board("sh688981") == Board.STAR assert detect_board("sh688981") == Board.STAR
assert detect_board("sz300750") == Board.CHINEXT assert detect_board("sz300750") == Board.CHINEXT
assert detect_board("bj830000") == Board.UNKNOWN assert detect_board("bj830000") == Board.UNKNOWN
assert detect_board("sh") == Board.UNKNOWN
# --- MarketRule date transitions --------------------------------------------- # --- MarketRule date transitions ---------------------------------------------
@@ -120,6 +126,26 @@ def test_get_rules_vectorized():
assert list(limit) == [0.10, 0.20, 0.20] assert list(limit) == [0.10, 0.20, 0.20]
def test_market_rule_date_coercion_unknown_board_and_st_vector_override():
rules = MarketRule()
assert _to_date(dt.datetime(2024, 1, 2, 15, 0)) == dt.date(2024, 1, 2)
assert _to_date(pd.Timestamp("2024-01-03 09:30")) == dt.date(2024, 1, 3)
assert _to_date("2024-01-04 10:00:00") == dt.date(2024, 1, 4)
unknown_rule = rules.get_rule("xx999999", "2024-01-02")
assert unknown_rule.minimum_open_size == 100
assert unknown_rule.share_increment == 100
assert unknown_rule.price_limit_pct == 0.10
_, _, _, limit = rules.get_rules_vectorized(
np.array(["sh600000", "sh600000"], dtype=object),
"2024-01-02",
np.array([0, 1]),
)
assert limit.tolist() == [0.10, 0.05]
def test_compute_limit_status(): def test_compute_limit_status():
price = np.array([110.0, 90.0, 100.0]) price = np.array([110.0, 90.0, 100.0])
preclose = np.array([100.0, 100.0, 100.0]) preclose = np.array([100.0, 100.0, 100.0])
@@ -130,6 +156,20 @@ def test_compute_limit_status():
assert status[2] == LimitStatus.NORMAL.value assert status[2] == LimitStatus.NORMAL.value
def test_compute_limit_status_treats_bad_preclose_as_normal():
status = compute_limit_status(
price=np.array([np.nan, 110.0, 90.0]),
preclose=np.array([100.0, np.nan, 0.0]),
limit_pct=np.array([0.10, 0.10, 0.10]),
)
assert status.tolist() == [
LimitStatus.NORMAL.value,
LimitStatus.NORMAL.value,
LimitStatus.NORMAL.value,
]
# --- continuous targets ------------------------------------------------------ # --- continuous targets ------------------------------------------------------
def test_continuous_targets_normalization(): def test_continuous_targets_normalization():
@@ -295,6 +335,70 @@ def test_repair_scales_to_4000_names():
assert abs(gross - B) <= 0.03 * B assert abs(gross - B) <= 0.03 * B
def test_repair_handles_empty_input():
result = repair_exposure(
np.array([], dtype=np.int64),
np.array([], dtype=float),
np.array([], dtype=float),
np.array([], dtype=np.int64),
np.array([], dtype=np.int64),
np.array([], dtype=np.int64),
)
assert result.dtype == np.int64
assert result.tolist() == []
def test_repair_respects_max_iters_and_zero_increment_noop():
q_round = np.array([100, -100], dtype=np.int64)
q_target = np.array([300.0, -300.0])
price = np.array([10.0, 10.0])
min_open = np.array([100, 100])
prev = np.zeros(2, dtype=np.int64)
capped = repair_exposure(
q_round,
q_target,
price,
increment=np.array([1, 1]),
min_open=min_open,
prev_shares=prev,
booksize=10_000.0,
gross_tol=0.0,
max_iters=0,
)
zero_increment = repair_exposure(
q_round,
q_target,
price,
increment=np.array([0, 0]),
min_open=min_open,
prev_shares=prev,
booksize=0.0,
net_tol=0.0,
gross_tol=0.0,
)
assert capped.tolist() == [100, -100]
assert zero_increment.tolist() == [100, -100]
def test_repair_gross_growth_obeys_net_band():
pos = repair_exposure(
q_round=np.array([100], dtype=np.int64),
q_target=np.array([300.0]),
price=np.array([10.0]),
increment=np.array([1]),
min_open=np.array([100]),
prev_shares=np.array([0], dtype=np.int64),
booksize=2_000.0,
net_tol=0.5,
gross_tol=0.0,
)
assert pos.tolist() == [100]
# --- construct_positions ----------------------------------------------------- # --- construct_positions -----------------------------------------------------
def test_construct_positions_schema(): def test_construct_positions_schema():
@@ -497,8 +601,81 @@ def test_constraints_compose_repeatably_regardless_of_order():
assert np.array_equal(first_order.cost, reversed_order.cost) assert np.array_equal(first_order.cost, reversed_order.cost)
def test_constraint_registry_and_default_adjust_targets():
class _NoopConstraint(TradeConstraint):
name = "_coverage_noop_constraint"
def delta_bounds(self, ctx):
return np.zeros(1), np.ones(1)
registered = register_constraint(_NoopConstraint)
assert registered is _NoopConstraint
assert "_coverage_noop_constraint" in available_constraints()
assert isinstance(get_constraint("_coverage_noop_constraint"), _NoopConstraint)
assert get_constraint("_coverage_noop_constraint").adjust_targets(object()) is None
with np.testing.assert_raises(KeyError):
get_constraint("_missing_constraint")
with np.testing.assert_raises(TypeError):
register_constraint(object) # type: ignore[arg-type]
with np.testing.assert_raises(ValueError):
class _NoNameConstraint(TradeConstraint):
def delta_bounds(self, ctx):
return np.zeros(1), np.ones(1)
register_constraint(_NoNameConstraint)
with np.testing.assert_raises(ValueError):
class _DuplicateConstraint(TradeConstraint):
name = "_coverage_noop_constraint"
def delta_bounds(self, ctx):
return np.zeros(1), np.ones(1)
register_constraint(_DuplicateConstraint)
# --- ReferenceSimulator ------------------------------------------------------ # --- ReferenceSimulator ------------------------------------------------------
def test_simulator_applies_constraint_target_adjustment():
class _HalveTarget(TradeConstraint):
name = "_halve_target"
def adjust_targets(self, ctx):
return ctx.target_shares // 2
def delta_bounds(self, ctx):
return np.full(len(ctx.target_shares), -np.inf), np.full(len(ctx.target_shares), np.inf)
sl = _slice(1, price=np.array([10.0]))
ctx = TradeContext(np.array([0], np.int64), np.array([100], np.int64), sl, 1e6)
result = ReferenceSimulator(constraints=[_HalveTarget()]).fill(ctx)
assert result.traded_shares.tolist() == [50]
assert result.realized_shares.tolist() == [50]
def test_simulator_empty_positions_uses_default_booksize():
data = pd.DataFrame({
"symbol_id": ["sh600000"],
"date": [pd.Timestamp("2024-01-02")],
"open": [10.0],
"close": [10.0],
"preclose": [10.0],
"amount": [1e9],
"tradestatus": [1],
"isST": [0],
})
positions = pd.DataFrame(columns=POSITION_COLUMNS)
fills, pnl = ReferenceSimulator().run(positions, data)
assert list(fills.columns) == FILL_COLUMNS
assert list(pnl.columns) == PNL_COLUMNS
assert fills.empty
assert pnl.empty
def test_simulator_next_open_and_blocked_buy_holds_prev(): def test_simulator_next_open_and_blocked_buy_holds_prev():
data = _make_data(n_days=15) data = _make_data(n_days=15)
weights = _make_weights(data) weights = _make_weights(data)
@@ -749,3 +926,34 @@ def test_evaluate_portfolio_excludes_signal_without_forward_return():
metrics = evaluate_portfolio(positions, data) metrics = evaluate_portfolio(positions, data)
assert metrics["n_dates"] == 1 assert metrics["n_dates"] == 1
def test_evaluate_portfolio_empty_and_single_return_paths():
empty_metrics = evaluate_portfolio(
pd.DataFrame(columns=POSITION_COLUMNS),
pd.DataFrame(columns=["symbol_id", "date", "open"]),
)
assert empty_metrics["n_dates"] == 0
assert empty_metrics["cumulative_return"] == 0.0
dates = pd.date_range("2024-01-01", periods=3)
data = pd.DataFrame([
{"symbol_id": "sh600000", "date": d, "open": price}
for d, price in zip(dates, [100.0, 100.0, 110.0])
])
positions = pd.DataFrame({
"symbol_id": ["sh600000"],
"date": [dates[0]],
"portfolio_name": ["single"],
"target_weight": [1.0],
"target_value": [1000.0],
"target_shares": [10.0],
"position_shares": [10],
"position_value": [1000.0],
"price": [100.0],
})
single_metrics = evaluate_portfolio(positions, data)
assert single_metrics["n_dates"] == 1
assert single_metrics["cumulative_return"] == 0.0