Add minute bar feature pipeline
This commit is contained in:
Yuxuan Yan
@@ -0,0 +1 @@
|
||||
"""Daily feature plugin package."""
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Base class for daily feature plugins."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class BaseFeature(ABC):
|
||||
"""Aggregate raw minute bars into daily, symbol-keyed feature columns."""
|
||||
|
||||
#: Unique registry key. Every concrete feature must set this to a non-empty str.
|
||||
name: str = ""
|
||||
|
||||
@abstractmethod
|
||||
def compute(
|
||||
self,
|
||||
minute: pd.DataFrame,
|
||||
daily: pd.DataFrame | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""Compute daily features.
|
||||
|
||||
Args:
|
||||
minute: Raw minute bars with ``symbol_id`` and ``date`` keys.
|
||||
daily: Optional daily data frame for calendar alignment or
|
||||
reference daily columns.
|
||||
|
||||
Returns:
|
||||
DataFrame with ``symbol_id``, ``date``, and one or more numeric
|
||||
feature columns.
|
||||
"""
|
||||
|
||||
def __repr__(self) -> str:
|
||||
params = ", ".join(f"{k}={v!r}" for k, v in vars(self).items())
|
||||
return f"{type(self).__name__}({params})"
|
||||
@@ -0,0 +1,108 @@
|
||||
"""CLI for daily feature computation."""
|
||||
|
||||
import os
|
||||
|
||||
import click
|
||||
import pandas as pd
|
||||
|
||||
from pipeline.features.compute import compute_feature
|
||||
from pipeline.features.registry import available_features, load_feature_module
|
||||
|
||||
|
||||
@click.group(name="feature")
|
||||
def feature():
|
||||
"""Compute daily feature parquet files from minute bars."""
|
||||
|
||||
|
||||
def _coerce(value: str):
|
||||
"""Best-effort coercion of a CLI string to int, then float, else str."""
|
||||
for cast in (int, float):
|
||||
try:
|
||||
return cast(value)
|
||||
except ValueError:
|
||||
continue
|
||||
return value
|
||||
|
||||
|
||||
def _parse_params(pairs: tuple[str, ...]) -> dict:
|
||||
"""Parse repeated ``name=value`` options into a params dict."""
|
||||
params: dict = {}
|
||||
for pair in pairs:
|
||||
if "=" not in pair:
|
||||
raise click.BadParameter(f"--param must be name=value, got '{pair}'")
|
||||
key, value = pair.split("=", 1)
|
||||
params[key.strip()] = _coerce(value.strip())
|
||||
return params
|
||||
|
||||
|
||||
@feature.command("list")
|
||||
@click.option(
|
||||
"--feature-module", "feature_modules", multiple=True,
|
||||
help="External module(s) to import first (dotted path or .py file)",
|
||||
)
|
||||
def list_(feature_modules):
|
||||
"""List the registered feature types."""
|
||||
for spec in feature_modules:
|
||||
load_feature_module(spec)
|
||||
for name in available_features():
|
||||
click.echo(name)
|
||||
|
||||
|
||||
@feature.command("compute")
|
||||
@click.option("--minute-path", required=True, help="Path to minute parquet dataset/file")
|
||||
@click.option("--daily-path", default=None, help="Optional daily data parquet for alignment")
|
||||
@click.option("--feature-type", required=True, help="Registry key of the feature class")
|
||||
@click.option("--feature-name", required=True, help="Name for this feature run/output file")
|
||||
@click.option("--output-dir", default="features", help="Directory to save feature parquet")
|
||||
@click.option(
|
||||
"--feature-module", "feature_modules", multiple=True,
|
||||
help="External module(s) to import so their features register (dotted path or .py file)",
|
||||
)
|
||||
@click.option(
|
||||
"--param", "extra_params", multiple=True,
|
||||
help="Extra feature constructor param as name=value (repeatable)",
|
||||
)
|
||||
def compute(
|
||||
minute_path,
|
||||
daily_path,
|
||||
feature_type,
|
||||
feature_name,
|
||||
output_dir,
|
||||
feature_modules,
|
||||
extra_params,
|
||||
):
|
||||
"""Compute one daily feature file from raw minute bars."""
|
||||
for spec in feature_modules:
|
||||
load_feature_module(spec)
|
||||
|
||||
options = available_features()
|
||||
if feature_type not in options:
|
||||
raise click.BadParameter(
|
||||
f"Unknown feature-type '{feature_type}'. Available: {options}. "
|
||||
f"Use --feature-module to register an external feature.",
|
||||
param_hint="--feature-type",
|
||||
)
|
||||
|
||||
minute = pd.read_parquet(minute_path)
|
||||
click.echo(f"Loaded minute bars: {len(minute):,} rows from {minute_path}")
|
||||
|
||||
daily = None
|
||||
if daily_path:
|
||||
daily = pd.read_parquet(daily_path)
|
||||
click.echo(f"Loaded daily data: {len(daily):,} rows from {daily_path}")
|
||||
|
||||
result = compute_feature(
|
||||
minute=minute,
|
||||
daily=daily,
|
||||
feature_type=feature_type,
|
||||
**_parse_params(extra_params),
|
||||
)
|
||||
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
out_path = f"{output_dir}/{feature_name}.pq"
|
||||
result.to_parquet(out_path, index=False)
|
||||
feature_cols = [col for col in result.columns if col not in ("symbol_id", "date")]
|
||||
click.echo(
|
||||
f"Saved feature: {out_path} ({len(result):,} rows, "
|
||||
f"{len(feature_cols)} columns)"
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Feature computation and validation."""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
import pandas as pd
|
||||
from pandas.api.types import is_numeric_dtype
|
||||
|
||||
from pipeline.features.registry import get_feature
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FEATURE_KEY_COLUMNS = ["symbol_id", "date"]
|
||||
|
||||
|
||||
def validate_feature_frame(features: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Validate and normalize a daily feature frame.
|
||||
|
||||
A valid feature frame is keyed by unique ``symbol_id,date`` rows and has at
|
||||
least one numeric feature column beyond those keys.
|
||||
"""
|
||||
duplicated = features.columns[features.columns.duplicated()].tolist()
|
||||
if duplicated:
|
||||
raise ValueError(f"Feature output has duplicate columns: {duplicated}")
|
||||
|
||||
missing = [col for col in FEATURE_KEY_COLUMNS if col not in features.columns]
|
||||
if missing:
|
||||
raise ValueError(f"Feature output missing required columns: {missing}")
|
||||
|
||||
out = features.copy()
|
||||
out["date"] = pd.to_datetime(out["date"])
|
||||
|
||||
if out.duplicated(FEATURE_KEY_COLUMNS).any():
|
||||
raise ValueError("Feature output has duplicate symbol_id,date rows")
|
||||
|
||||
feature_cols = [col for col in out.columns if col not in FEATURE_KEY_COLUMNS]
|
||||
if not feature_cols:
|
||||
raise ValueError("Feature output must include at least one feature column")
|
||||
|
||||
non_numeric = [col for col in feature_cols if not is_numeric_dtype(out[col])]
|
||||
if non_numeric:
|
||||
raise ValueError(f"Feature columns must be numeric: {non_numeric}")
|
||||
|
||||
out = out[FEATURE_KEY_COLUMNS + feature_cols].copy()
|
||||
return out.sort_values(FEATURE_KEY_COLUMNS).reset_index(drop=True)
|
||||
|
||||
|
||||
def compute_feature(
|
||||
minute: pd.DataFrame,
|
||||
feature_type: str,
|
||||
daily: pd.DataFrame | None = None,
|
||||
**params,
|
||||
) -> pd.DataFrame:
|
||||
"""Compute one registered feature from raw minute bars."""
|
||||
feature = get_feature(feature_type, **params)
|
||||
result = validate_feature_frame(feature.compute(minute=minute, daily=daily))
|
||||
feature_cols = [col for col in result.columns if col not in FEATURE_KEY_COLUMNS]
|
||||
logger.info(
|
||||
"Feature '%s' (%r): %d symbols × %d dates, columns=%s",
|
||||
feature_type,
|
||||
feature,
|
||||
result["symbol_id"].nunique(),
|
||||
result["date"].nunique(),
|
||||
feature_cols,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def read_feature_frames(feature_paths: Iterable[str | Path]) -> list[pd.DataFrame]:
|
||||
"""Read and validate feature parquet files."""
|
||||
return [
|
||||
validate_feature_frame(pd.read_parquet(path))
|
||||
for path in feature_paths
|
||||
]
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Built-in feature library."""
|
||||
|
||||
from pipeline.features.library import minute_daily_summary # noqa: F401
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Daily summary features derived from raw minute bars."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from pipeline.features.base import BaseFeature
|
||||
from pipeline.features.registry import register_feature
|
||||
|
||||
|
||||
@register_feature
|
||||
class MinuteDailySummaryFeature(BaseFeature):
|
||||
"""Aggregate intraday bars into daily summary columns."""
|
||||
|
||||
name = "minute_daily_summary"
|
||||
|
||||
def compute(
|
||||
self,
|
||||
minute: pd.DataFrame,
|
||||
daily: pd.DataFrame | None = None,
|
||||
) -> pd.DataFrame:
|
||||
minute = minute.copy()
|
||||
minute["date"] = pd.to_datetime(minute["date"])
|
||||
sort_cols = ["symbol_id", "date"]
|
||||
if "datetime" in minute.columns:
|
||||
minute["datetime"] = pd.to_datetime(minute["datetime"])
|
||||
sort_cols.append("datetime")
|
||||
elif "time" in minute.columns:
|
||||
sort_cols.append("time")
|
||||
minute = minute.sort_values(sort_cols)
|
||||
|
||||
grouped = minute.groupby(["symbol_id", "date"], sort=True)
|
||||
summary = grouped.agg(
|
||||
minute_bar_count=("close", "count"),
|
||||
first_open=("open", "first"),
|
||||
last_close=("close", "last"),
|
||||
high=("high", "max"),
|
||||
low=("low", "min"),
|
||||
volume_sum=("volume", "sum"),
|
||||
amount_sum=("amount", "sum"),
|
||||
)
|
||||
summary["minute_intraday_return"] = (
|
||||
summary["last_close"] / summary["first_open"] - 1.0
|
||||
)
|
||||
summary["minute_intraday_range"] = summary["high"] / summary["low"] - 1.0
|
||||
summary["minute_vwap"] = (
|
||||
summary["amount_sum"] / summary["volume_sum"].where(summary["volume_sum"] > 0)
|
||||
)
|
||||
summary = summary.reset_index()
|
||||
|
||||
if daily is not None:
|
||||
daily_keys = daily[["symbol_id", "date"]].copy()
|
||||
daily_keys["date"] = pd.to_datetime(daily_keys["date"])
|
||||
daily_keys = daily_keys.drop_duplicates(["symbol_id", "date"])
|
||||
result = daily_keys.merge(summary, on=["symbol_id", "date"], how="left")
|
||||
if "close" in daily.columns:
|
||||
daily_close = daily[["symbol_id", "date", "close"]].copy()
|
||||
daily_close["date"] = pd.to_datetime(daily_close["date"])
|
||||
daily_close = daily_close.drop_duplicates(["symbol_id", "date"])
|
||||
result = result.merge(
|
||||
daily_close.rename(columns={"close": "daily_close"}),
|
||||
on=["symbol_id", "date"],
|
||||
how="left",
|
||||
)
|
||||
reference_close = result["daily_close"].fillna(result["last_close"])
|
||||
else:
|
||||
reference_close = result["last_close"]
|
||||
else:
|
||||
result = summary
|
||||
reference_close = result["last_close"]
|
||||
|
||||
result["minute_vwap_deviation"] = (
|
||||
result["minute_vwap"] / reference_close.replace(0.0, np.nan) - 1.0
|
||||
)
|
||||
return result[
|
||||
[
|
||||
"symbol_id",
|
||||
"date",
|
||||
"minute_bar_count",
|
||||
"minute_intraday_return",
|
||||
"minute_intraday_range",
|
||||
"minute_vwap",
|
||||
"minute_vwap_deviation",
|
||||
]
|
||||
]
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Registry and factory for daily feature plugins."""
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from typing import Optional, Type
|
||||
|
||||
from pipeline.features.base import BaseFeature
|
||||
|
||||
_REGISTRY: dict[str, Type[BaseFeature]] = {}
|
||||
_builtins_loaded = False
|
||||
|
||||
|
||||
def register_feature(cls: Type[BaseFeature]) -> Type[BaseFeature]:
|
||||
"""Class decorator that registers a feature under ``BaseFeature.name``."""
|
||||
if not (isinstance(cls, type) and issubclass(cls, BaseFeature)):
|
||||
raise TypeError(f"{cls!r} is not a BaseFeature subclass")
|
||||
key = getattr(cls, "name", "")
|
||||
if not key:
|
||||
raise ValueError(f"{cls.__name__} must set a non-empty class attribute `name`")
|
||||
existing = _REGISTRY.get(key)
|
||||
if existing is not None and existing is not cls:
|
||||
raise ValueError(
|
||||
f"Feature name '{key}' already registered by {existing.__name__}"
|
||||
)
|
||||
_REGISTRY[key] = cls
|
||||
return cls
|
||||
|
||||
|
||||
def available_features() -> list[str]:
|
||||
"""Sorted names of all registered features (built-ins are loaded lazily)."""
|
||||
_ensure_builtins()
|
||||
return sorted(_REGISTRY)
|
||||
|
||||
|
||||
def get_feature(name: str, **params) -> BaseFeature:
|
||||
"""Instantiate a registered feature by name.
|
||||
|
||||
Only parameters accepted by the feature class's ``__init__`` are forwarded.
|
||||
"""
|
||||
_ensure_builtins()
|
||||
if name not in _REGISTRY:
|
||||
raise KeyError(f"Unknown feature '{name}'. Available: {sorted(_REGISTRY)}")
|
||||
cls = _REGISTRY[name]
|
||||
accepted = _accepted_params(cls)
|
||||
kwargs = params if accepted is None else {k: v for k, v in params.items() if k in accepted}
|
||||
return cls(**kwargs)
|
||||
|
||||
|
||||
def load_feature_module(spec: str) -> None:
|
||||
"""Import an external module so its ``@register_feature`` classes register."""
|
||||
looks_like_file = spec.endswith(".py") or Path(spec).expanduser().exists()
|
||||
if looks_like_file:
|
||||
path = Path(spec).expanduser().resolve()
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Feature module not found: {path}")
|
||||
module_spec = importlib.util.spec_from_file_location(path.stem, path)
|
||||
if module_spec is None or module_spec.loader is None:
|
||||
raise ImportError(f"Cannot load feature module from {path}")
|
||||
module = importlib.util.module_from_spec(module_spec)
|
||||
module_spec.loader.exec_module(module)
|
||||
else:
|
||||
importlib.import_module(spec)
|
||||
|
||||
|
||||
def _accepted_params(cls: Type[BaseFeature]) -> Optional[set[str]]:
|
||||
"""Param names ``cls.__init__`` accepts, or None if it takes ``**kwargs``."""
|
||||
sig = inspect.signature(cls.__init__)
|
||||
if any(p.kind is p.VAR_KEYWORD for p in sig.parameters.values()):
|
||||
return None
|
||||
return {name for name in sig.parameters if name != "self"}
|
||||
|
||||
|
||||
def _ensure_builtins() -> None:
|
||||
global _builtins_loaded
|
||||
if not _builtins_loaded:
|
||||
import pipeline.features.library # noqa: F401
|
||||
_builtins_loaded = True
|
||||
Reference in New Issue
Block a user