Add daily derived data pipeline

2026-06-16 15:55:30 +08:00
parent 83a006bbe4
commit 8d908477e2
19 changed files with 897 additions and 231 deletions
@@ -0,0 +1,88 @@
+"""Daily summary data derived from raw minute bars."""
+
+import numpy as np
+import pandas as pd
+
+from pipeline.derived.base import BaseDerivedData
+from pipeline.derived.registry import register_derived
+
+
+@register_derived
+class MinuteDailySummaryDerived(BaseDerivedData):
+    """Aggregate intraday bars into daily summary columns."""
+
+    name = "minute_daily_summary"
+
+    def compute(
+        self,
+        daily: pd.DataFrame | None = None,
+        minute: pd.DataFrame | None = None,
+    ) -> pd.DataFrame:
+        if minute is None:
+            raise ValueError("minute_daily_summary requires minute input")
+
+        minute = minute.copy()
+        minute["date"] = pd.to_datetime(minute["date"]).dt.normalize()
+        sort_cols = ["symbol_id", "date"]
+        if "datetime" in minute.columns:
+            minute["datetime"] = pd.to_datetime(minute["datetime"])
+            sort_cols.append("datetime")
+        elif "time" in minute.columns:
+            sort_cols.append("time")
+        minute = minute.sort_values(sort_cols)
+
+        grouped = minute.groupby(["symbol_id", "date"], sort=True)
+        summary = grouped.agg(
+            minute_bar_count=("close", "count"),
+            first_open=("open", "first"),
+            last_close=("close", "last"),
+            high=("high", "max"),
+            low=("low", "min"),
+            volume_sum=("volume", "sum"),
+            amount_sum=("amount", "sum"),
+        )
+        summary["minute_intraday_return"] = (
+            summary["last_close"] / summary["first_open"] - 1.0
+        )
+        summary["minute_intraday_range"] = summary["high"] / summary["low"] - 1.0
+        summary["minute_vwap"] = (
+            summary["amount_sum"] / summary["volume_sum"].where(summary["volume_sum"] > 0)
+        )
+        summary = summary.reset_index()
+
+        if daily is not None:
+            daily_keys = daily[["symbol_id", "date"]].copy()
+            daily_keys["date"] = pd.to_datetime(daily_keys["date"]).dt.normalize()
+            daily_keys = daily_keys.drop_duplicates(["symbol_id", "date"])
+            result = daily_keys.merge(summary, on=["symbol_id", "date"], how="left")
+            if "close" in daily.columns:
+                daily_close = daily[["symbol_id", "date", "close"]].copy()
+                daily_close["date"] = pd.to_datetime(daily_close["date"]).dt.normalize()
+                daily_close = daily_close.drop_duplicates(["symbol_id", "date"])
+                result = result.merge(
+                    daily_close.rename(columns={"close": "daily_close"}),
+                    on=["symbol_id", "date"],
+                    how="left",
+                )
+                reference_close = result["daily_close"].fillna(result["last_close"])
+            else:
+                reference_close = result["last_close"]
+        else:
+            result = summary
+            reference_close = result["last_close"]
+
+        result["minute_vwap_deviation"] = (
+            result["minute_vwap"] / reference_close.replace(0.0, np.nan) - 1.0
+        )
+        return result[
+            [
+                "symbol_id",
+                "date",
+                "minute_bar_count",
+                "minute_intraday_return",
+                "minute_intraday_range",
+                "minute_vwap",
+                "minute_vwap_deviation",
+            ]
+        ]
+