Add daily derived data pipeline

This commit is contained in:
Yuxuan Yan
2026-06-16 15:55:30 +08:00
parent 83a006bbe4
commit 8d908477e2
19 changed files with 897 additions and 231 deletions
+1 -1
View File
@@ -58,7 +58,7 @@ def list_(alpha_modules):
@click.option("--vol-window", default=20, type=int, help="Volatility window (reversal_vol only)")
@click.option(
"--feature-path", "feature_paths", multiple=True,
help="Daily feature parquet file/dataset to left-join on symbol_id,date (repeatable)",
help="Daily derived/feature parquet file or dataset to left-join on symbol_id,date (repeatable)",
)
@click.option(
"--alpha-module", "alpha_modules", multiple=True,
+10 -6
View File
@@ -15,7 +15,11 @@ import pandas as pd
from pipeline.alpha.registry import get_alpha
from pipeline.common.schema import ALPHA_COLUMNS
from pipeline.features.compute import FEATURE_KEY_COLUMNS, read_feature_frames, validate_feature_frame
from pipeline.derived.compute import (
DERIVED_KEY_COLUMNS,
read_derived_frames,
validate_derived_frame,
)
logger = logging.getLogger(__name__)
@@ -40,15 +44,15 @@ def join_feature_frames(
data: pd.DataFrame,
feature_frames: Iterable[pd.DataFrame],
) -> pd.DataFrame:
"""Left-join validated daily feature frames onto long daily data."""
"""Left-join validated daily derived/feature frames onto long daily data."""
out = data.copy()
out["date"] = pd.to_datetime(out["date"])
existing = set(out.columns)
joined_cols: list[str] = []
for frame in feature_frames:
features = validate_feature_frame(frame)
feature_cols = [col for col in features.columns if col not in FEATURE_KEY_COLUMNS]
features = validate_derived_frame(frame)
feature_cols = [col for col in features.columns if col not in DERIVED_KEY_COLUMNS]
overlap = sorted(existing.intersection(feature_cols))
if overlap:
raise ValueError(
@@ -56,7 +60,7 @@ def join_feature_frames(
)
out = out.merge(
features,
on=FEATURE_KEY_COLUMNS,
on=DERIVED_KEY_COLUMNS,
how="left",
validate="many_to_one",
)
@@ -171,7 +175,7 @@ def compute_alpha(
"""
feature_inputs: list[pd.DataFrame] = []
if feature_paths:
feature_inputs.extend(read_feature_frames(feature_paths))
feature_inputs.extend(read_derived_frames(feature_paths))
if feature_frames:
feature_inputs.extend(feature_frames)
if feature_inputs: