Add daily derived data pipeline

2026-06-16 15:55:30 +08:00
parent 83a006bbe4
commit 8d908477e2
19 changed files with 897 additions and 231 deletions
@@ -0,0 +1,2 @@
+"""Daily derived-data plugin package."""
+
@@ -0,0 +1,38 @@
+"""Base class for daily derived-data plugins."""
+
+from abc import ABC, abstractmethod
+
+import pandas as pd
+
+
+class BaseDerivedData(ABC):
+    """Compute daily, symbol-keyed numeric derived data.
+
+    Derived-data plugins may use daily bars, minute bars, or both as inputs, but
+    they must always return daily rows keyed by ``symbol_id,date``.
+    """
+
+    #: Unique registry key. Every concrete derived-data plugin must set this.
+    name: str = ""
+
+    @abstractmethod
+    def compute(
+        self,
+        daily: pd.DataFrame | None = None,
+        minute: pd.DataFrame | None = None,
+    ) -> pd.DataFrame:
+        """Compute daily derived data.
+
+        Args:
+            daily: Optional daily market data.
+            minute: Optional raw minute bars.
+
+        Returns:
+            DataFrame with ``symbol_id``, ``date``, and one or more numeric
+            derived-data columns.
+        """
+
+    def __repr__(self) -> str:
+        params = ", ".join(f"{k}={v!r}" for k, v in vars(self).items())
+        return f"{type(self).__name__}({params})"
+
@@ -0,0 +1,145 @@
+"""CLI for daily derived-data ingestion and computation."""
+
+import click
+import pandas as pd
+
+from pipeline.derived.compute import (
+    DERIVED_KEY_COLUMNS,
+    compute_derived,
+    read_derived_frame,
+    write_derived_frame,
+)
+from pipeline.derived.registry import (
+    available_derived,
+    load_derived_module,
+)
+
+
+@click.group(name="derived")
+def derived():
+    """Ingest, compute, and validate daily derived data."""
+
+
+def _coerce(value: str):
+    """Best-effort coercion of a CLI string to int, then float, else str."""
+    for cast in (int, float):
+        try:
+            return cast(value)
+        except ValueError:
+            continue
+    return value
+
+
+def _parse_params(pairs: tuple[str, ...]) -> dict:
+    """Parse repeated ``name=value`` options into a params dict."""
+    params: dict = {}
+    for pair in pairs:
+        if "=" not in pair:
+            raise click.BadParameter(f"--param must be name=value, got '{pair}'")
+        key, value = pair.split("=", 1)
+        params[key.strip()] = _coerce(value.strip())
+    return params
+
+
+def _read_optional_parquet(path: str | None) -> pd.DataFrame | None:
+    return None if path is None else pd.read_parquet(path)
+
+
+def _summarize(result: pd.DataFrame) -> str:
+    value_cols = [col for col in result.columns if col not in DERIVED_KEY_COLUMNS]
+    return f"{len(result):,} rows, {len(value_cols)} columns"
+
+
+@derived.command("list")
+@click.option(
+    "--derived-module", "derived_modules", multiple=True,
+    help="External module(s) to import first (dotted path or .py file)",
+)
+def list_(derived_modules):
+    """List the registered derived-data plugin types."""
+    for spec in derived_modules:
+        load_derived_module(spec)
+    for name in available_derived():
+        click.echo(name)
+
+
+@derived.command("validate")
+@click.option("--input-path", required=True, help="CSV/parquet file or parquet dataset to validate")
+def validate(input_path):
+    """Validate a daily derived-data file without writing output."""
+    try:
+        result = read_derived_frame(input_path)
+    except Exception as exc:
+        raise click.ClickException(str(exc)) from exc
+    click.echo(f"Valid derived data: {input_path}  ({_summarize(result)})")
+
+
+@derived.command("ingest")
+@click.option("--input-path", required=True, help="CSV/parquet file to ingest")
+@click.option("--derived-name", required=True, help="Name for this derived-data output file")
+@click.option("--output-dir", default="derived", help="Directory to save derived parquet")
+def ingest(input_path, derived_name, output_dir):
+    """Ingest a user-provided daily derived-data CSV/parquet file."""
+    try:
+        result = read_derived_frame(input_path)
+        out_path = write_derived_frame(result, derived_name, output_dir=output_dir)
+    except Exception as exc:
+        raise click.ClickException(str(exc)) from exc
+    click.echo(f"Saved derived data: {out_path}  ({_summarize(result)})")
+
+
+@derived.command("compute")
+@click.option("--daily-path", default=None, help="Optional daily data parquet/dataset")
+@click.option("--minute-path", default=None, help="Optional minute parquet/dataset")
+@click.option("--derived-type", required=True, help="Registry key of the derived-data plugin")
+@click.option("--derived-name", required=True, help="Name for this derived-data output file")
+@click.option("--output-dir", default="derived", help="Directory to save derived parquet")
+@click.option(
+    "--derived-module", "derived_modules", multiple=True,
+    help="External module(s) to import so their derived-data plugins register",
+)
+@click.option(
+    "--param", "extra_params", multiple=True,
+    help="Extra derived-data constructor param as name=value (repeatable)",
+)
+def compute(
+    daily_path,
+    minute_path,
+    derived_type,
+    derived_name,
+    output_dir,
+    derived_modules,
+    extra_params,
+):
+    """Compute one daily derived-data file from daily and/or minute inputs."""
+    for spec in derived_modules:
+        load_derived_module(spec)
+
+    options = available_derived()
+    if derived_type not in options:
+        raise click.BadParameter(
+            f"Unknown derived-type '{derived_type}'. Available: {options}. "
+            f"Use --derived-module to register an external derived-data plugin.",
+            param_hint="--derived-type",
+        )
+    if daily_path is None and minute_path is None:
+        raise click.UsageError("At least one of --daily-path or --minute-path is required")
+
+    daily = _read_optional_parquet(daily_path)
+    if daily_path:
+        click.echo(f"Loaded daily data: {len(daily):,} rows from {daily_path}")
+    minute = _read_optional_parquet(minute_path)
+    if minute_path:
+        click.echo(f"Loaded minute bars: {len(minute):,} rows from {minute_path}")
+
+    try:
+        result = compute_derived(
+            derived_type=derived_type,
+            daily=daily,
+            minute=minute,
+            **_parse_params(extra_params),
+        )
+        out_path = write_derived_frame(result, derived_name, output_dir=output_dir)
+    except Exception as exc:
+        raise click.ClickException(str(exc)) from exc
+    click.echo(f"Saved derived data: {out_path}  ({_summarize(result)})")
@@ -0,0 +1,115 @@
+"""Derived-data computation and validation."""
+
+import csv
+import logging
+from pathlib import Path
+from typing import Iterable
+
+import pandas as pd
+from pandas.api.types import is_bool_dtype, is_numeric_dtype
+
+from pipeline.common.schema import DERIVED_KEY_COLUMNS
+from pipeline.derived.registry import get_derived
+
+logger = logging.getLogger(__name__)
+
+
+def validate_derived_frame(derived: pd.DataFrame) -> pd.DataFrame:
+    """Validate and normalize a daily derived-data frame.
+
+    A valid derived frame is keyed by unique ``symbol_id,date`` rows and has at
+    least one numeric value column beyond those keys. Dates are normalized to
+    daily timestamps before duplicate-key checks.
+    """
+    duplicated = derived.columns[derived.columns.duplicated()].tolist()
+    if duplicated:
+        raise ValueError(f"Derived data has duplicate columns: {duplicated}")
+
+    missing = [col for col in DERIVED_KEY_COLUMNS if col not in derived.columns]
+    if missing:
+        raise ValueError(f"Derived data missing required columns: {missing}")
+
+    out = derived.copy()
+    out["date"] = pd.to_datetime(out["date"]).dt.normalize()
+
+    if out.duplicated(DERIVED_KEY_COLUMNS).any():
+        raise ValueError("Derived data has duplicate symbol_id,date rows")
+
+    value_cols = [col for col in out.columns if col not in DERIVED_KEY_COLUMNS]
+    if not value_cols:
+        raise ValueError("Derived data must include at least one value column")
+
+    non_numeric = [
+        col
+        for col in value_cols
+        if is_bool_dtype(out[col]) or not is_numeric_dtype(out[col])
+    ]
+    if non_numeric:
+        raise ValueError(f"Derived data value columns must be numeric: {non_numeric}")
+
+    out = out[DERIVED_KEY_COLUMNS + value_cols].copy()
+    return out.sort_values(DERIVED_KEY_COLUMNS).reset_index(drop=True)
+
+
+def compute_derived(
+    derived_type: str,
+    daily: pd.DataFrame | None = None,
+    minute: pd.DataFrame | None = None,
+    **params,
+) -> pd.DataFrame:
+    """Compute one registered derived-data plugin."""
+    if daily is None and minute is None:
+        raise ValueError("Derived data computation requires --daily-path or --minute-path")
+
+    derived = get_derived(derived_type, **params)
+    result = validate_derived_frame(derived.compute(daily=daily, minute=minute))
+    value_cols = [col for col in result.columns if col not in DERIVED_KEY_COLUMNS]
+    logger.info(
+        "Derived data '%s' (%r): %d symbols × %d dates, columns=%s",
+        derived_type,
+        derived,
+        result["symbol_id"].nunique(),
+        result["date"].nunique(),
+        value_cols,
+    )
+    return result
+
+
+def read_derived_frame(path: str | Path) -> pd.DataFrame:
+    """Read and validate one derived CSV/parquet file or parquet dataset."""
+    path = Path(path)
+    if path.suffix.lower() == ".csv":
+        return validate_derived_frame(_read_csv_with_duplicate_header_check(path))
+    return validate_derived_frame(pd.read_parquet(path))
+
+
+def read_derived_frames(derived_paths: Iterable[str | Path]) -> list[pd.DataFrame]:
+    """Read and validate derived-data files."""
+    return [read_derived_frame(path) for path in derived_paths]
+
+
+def write_derived_frame(
+    derived: pd.DataFrame,
+    derived_name: str,
+    output_dir: str | Path = "derived",
+) -> Path:
+    """Validate and write derived data to ``{output_dir}/{derived_name}.pq``."""
+    result = validate_derived_frame(derived)
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    out_path = output_dir / f"{derived_name}.pq"
+    result.to_parquet(out_path, index=False)
+    return out_path
+
+
+def _read_csv_with_duplicate_header_check(path: Path) -> pd.DataFrame:
+    with path.open(newline="") as fh:
+        reader = csv.reader(fh)
+        try:
+            header = next(reader)
+        except StopIteration as exc:
+            raise ValueError("CSV input is empty") from exc
+    duplicated = sorted({col for col in header if header.count(col) > 1})
+    if duplicated:
+        raise ValueError(f"Derived data has duplicate columns: {duplicated}")
+    return pd.read_csv(path)
@@ -0,0 +1,4 @@
+"""Built-in derived-data library."""
+
+from pipeline.derived.library import minute_daily_summary  # noqa: F401
+
@@ -0,0 +1,88 @@
+"""Daily summary data derived from raw minute bars."""
+
+import numpy as np
+import pandas as pd
+
+from pipeline.derived.base import BaseDerivedData
+from pipeline.derived.registry import register_derived
+
+
+@register_derived
+class MinuteDailySummaryDerived(BaseDerivedData):
+    """Aggregate intraday bars into daily summary columns."""
+
+    name = "minute_daily_summary"
+
+    def compute(
+        self,
+        daily: pd.DataFrame | None = None,
+        minute: pd.DataFrame | None = None,
+    ) -> pd.DataFrame:
+        if minute is None:
+            raise ValueError("minute_daily_summary requires minute input")
+
+        minute = minute.copy()
+        minute["date"] = pd.to_datetime(minute["date"]).dt.normalize()
+        sort_cols = ["symbol_id", "date"]
+        if "datetime" in minute.columns:
+            minute["datetime"] = pd.to_datetime(minute["datetime"])
+            sort_cols.append("datetime")
+        elif "time" in minute.columns:
+            sort_cols.append("time")
+        minute = minute.sort_values(sort_cols)
+
+        grouped = minute.groupby(["symbol_id", "date"], sort=True)
+        summary = grouped.agg(
+            minute_bar_count=("close", "count"),
+            first_open=("open", "first"),
+            last_close=("close", "last"),
+            high=("high", "max"),
+            low=("low", "min"),
+            volume_sum=("volume", "sum"),
+            amount_sum=("amount", "sum"),
+        )
+        summary["minute_intraday_return"] = (
+            summary["last_close"] / summary["first_open"] - 1.0
+        )
+        summary["minute_intraday_range"] = summary["high"] / summary["low"] - 1.0
+        summary["minute_vwap"] = (
+            summary["amount_sum"] / summary["volume_sum"].where(summary["volume_sum"] > 0)
+        )
+        summary = summary.reset_index()
+
+        if daily is not None:
+            daily_keys = daily[["symbol_id", "date"]].copy()
+            daily_keys["date"] = pd.to_datetime(daily_keys["date"]).dt.normalize()
+            daily_keys = daily_keys.drop_duplicates(["symbol_id", "date"])
+            result = daily_keys.merge(summary, on=["symbol_id", "date"], how="left")
+            if "close" in daily.columns:
+                daily_close = daily[["symbol_id", "date", "close"]].copy()
+                daily_close["date"] = pd.to_datetime(daily_close["date"]).dt.normalize()
+                daily_close = daily_close.drop_duplicates(["symbol_id", "date"])
+                result = result.merge(
+                    daily_close.rename(columns={"close": "daily_close"}),
+                    on=["symbol_id", "date"],
+                    how="left",
+                )
+                reference_close = result["daily_close"].fillna(result["last_close"])
+            else:
+                reference_close = result["last_close"]
+        else:
+            result = summary
+            reference_close = result["last_close"]
+
+        result["minute_vwap_deviation"] = (
+            result["minute_vwap"] / reference_close.replace(0.0, np.nan) - 1.0
+        )
+        return result[
+            [
+                "symbol_id",
+                "date",
+                "minute_bar_count",
+                "minute_intraday_return",
+                "minute_intraday_range",
+                "minute_vwap",
+                "minute_vwap_deviation",
+            ]
+        ]
+
@@ -0,0 +1,80 @@
+"""Registry and factory for daily derived-data plugins."""
+
+import importlib
+import importlib.util
+import inspect
+from pathlib import Path
+from typing import Optional, Type
+
+from pipeline.derived.base import BaseDerivedData
+
+_REGISTRY: dict[str, Type[BaseDerivedData]] = {}
+_builtins_loaded = False
+
+
+def register_derived(cls: Type[BaseDerivedData]) -> Type[BaseDerivedData]:
+    """Class decorator that registers derived data under ``BaseDerivedData.name``."""
+    if not (isinstance(cls, type) and issubclass(cls, BaseDerivedData)):
+        raise TypeError(f"{cls!r} is not a BaseDerivedData subclass")
+    key = getattr(cls, "name", "")
+    if not key:
+        raise ValueError(f"{cls.__name__} must set a non-empty class attribute `name`")
+    existing = _REGISTRY.get(key)
+    if existing is not None and existing is not cls:
+        raise ValueError(
+            f"Derived data name '{key}' already registered by {existing.__name__}"
+        )
+    _REGISTRY[key] = cls
+    return cls
+
+
+def available_derived() -> list[str]:
+    """Sorted names of all registered derived-data plugins."""
+    _ensure_builtins()
+    return sorted(_REGISTRY)
+
+
+def get_derived(name: str, **params) -> BaseDerivedData:
+    """Instantiate a registered derived-data plugin by name.
+
+    Only parameters accepted by the plugin class's ``__init__`` are forwarded.
+    """
+    _ensure_builtins()
+    if name not in _REGISTRY:
+        raise KeyError(f"Unknown derived data '{name}'. Available: {sorted(_REGISTRY)}")
+    cls = _REGISTRY[name]
+    accepted = _accepted_params(cls)
+    kwargs = params if accepted is None else {k: v for k, v in params.items() if k in accepted}
+    return cls(**kwargs)
+
+
+def load_derived_module(spec: str) -> None:
+    """Import an external module so its ``@register_derived`` classes register."""
+    looks_like_file = spec.endswith(".py") or Path(spec).expanduser().exists()
+    if looks_like_file:
+        path = Path(spec).expanduser().resolve()
+        if not path.exists():
+            raise FileNotFoundError(f"Derived data module not found: {path}")
+        module_spec = importlib.util.spec_from_file_location(path.stem, path)
+        if module_spec is None or module_spec.loader is None:
+            raise ImportError(f"Cannot load derived data module from {path}")
+        module = importlib.util.module_from_spec(module_spec)
+        module_spec.loader.exec_module(module)
+    else:
+        importlib.import_module(spec)
+
+
+def _accepted_params(cls: Type[BaseDerivedData]) -> Optional[set[str]]:
+    """Param names ``cls.__init__`` accepts, or None if it takes ``**kwargs``."""
+    sig = inspect.signature(cls.__init__)
+    if any(p.kind is p.VAR_KEYWORD for p in sig.parameters.values()):
+        return None
+    return {name for name in sig.parameters if name != "self"}
+
+
+def _ensure_builtins() -> None:
+    global _builtins_loaded
+    if not _builtins_loaded:
+        import pipeline.derived.library  # noqa: F401
+        _builtins_loaded = True
+