Add minute bar feature pipeline

data/downloader.py

@@ -31,11 +31,68 @@ _BATCH_COLUMNS = [
     "peTTM", "pbMRQ", "psTTM", "pcfNcfTTM",
 ]
 
+# Raw Baostock minute bars. The ``time`` field is usually compact
+# YYYYMMDDHHMMSSmmm; parsing below also tolerates HH:MM:SS strings in tests.
+_MINUTE_FIELDS = "date,time,code,open,high,low,close,volume,amount,adjustflag"
+_MINUTE_NUMERIC = ["open", "high", "low", "close", "volume", "amount"]
+_MINUTE_COLUMNS = [
+    "symbol", "datetime", "date", "time", "frequency",
+    "open", "high", "low", "close", "volume", "amount", "vwap", "adjustflag",
+]
+_MINUTE_FREQUENCIES = {"5", "15", "30", "60"}
+
 
 class _SessionLost(Exception):
     """baostock reported the session was dropped (``用户未登录``)."""
 
 
+def _normalize_minute_frequency(frequency: str | int) -> tuple[str, str]:
+    """Return Baostock frequency and partition label for a minute interval."""
+    raw = str(frequency).strip().lower()
+    if raw.endswith("m"):
+        raw = raw[:-1]
+    if raw not in _MINUTE_FREQUENCIES:
+        raise ValueError(
+            f"Unsupported minute frequency '{frequency}'. "
+            f"Expected one of {sorted(_MINUTE_FREQUENCIES)} minutes."
+        )
+    return raw, f"{raw}m"
+
+
+def _parse_minute_datetime(date: pd.Series, time: pd.Series) -> pd.Series:
+    """Parse Baostock minute timestamps into pandas datetimes."""
+    date_dt = pd.to_datetime(date, errors="coerce")
+    date_compact = date_dt.dt.strftime("%Y%m%d")
+    time_text = time.astype(str).str.strip()
+    time_digits = time_text.str.replace(r"\D", "", regex=True)
+
+    full_digits = time_digits.str.slice(0, 14)
+    from_full = pd.to_datetime(full_digits, format="%Y%m%d%H%M%S", errors="coerce")
+
+    from_short = pd.Series(pd.NaT, index=date.index, dtype="datetime64[ns]")
+    short_time = time_digits.str.len().between(1, 6)
+    if short_time.any():
+        short_digits = (
+            time_digits.loc[short_time]
+            .str.pad(6, side="right", fillchar="0")
+            .str.slice(0, 6)
+        )
+        from_short.loc[short_time] = pd.to_datetime(
+            date_compact.loc[short_time] + short_digits,
+            format="%Y%m%d%H%M%S",
+            errors="coerce",
+        )
+
+    from_text = pd.Series(pd.NaT, index=date.index, dtype="datetime64[ns]")
+    text_time = time_text.str.contains(":", regex=False)
+    if text_time.any():
+        from_text.loc[text_time] = pd.to_datetime(
+            date.astype(str).loc[text_time] + " " + time_text.loc[text_time],
+            errors="coerce",
+        )
+    return from_full.fillna(from_short).fillna(from_text)
+
+
 def _download_akshare(symbol: str, start: str, end: str, adjust: str = "qfq") -> Optional[pd.DataFrame]:
     """Download daily bars from akshare. Returns DataFrame with OHLCV columns."""
     try:
@@ -239,3 +296,104 @@ def download_daily_batch(
         except Exception:
             pass
 
+
+def download_minute_batch(
+    symbols: Iterable[str],
+    start: str,
+    end: str,
+    frequency: str | int = 5,
+    relogin_every: int = 200,
+) -> Iterator[Tuple[str, Optional[pd.DataFrame]]]:
+    """Download raw Baostock minute bars for many symbols.
+
+    Minute bars are intentionally unadjusted (`adjustflag='3'`) because the
+    output is raw intraday market data for downstream feature aggregation, not a
+    tradable daily price series.
+
+    Args:
+        symbols: Internal-form symbols (``sh600000`` / ``sz000001``).
+        start, end: ``YYYY-MM-DD`` bounds.
+        frequency: Baostock minute frequency. ``5``/``"5"``/``"5m"`` all mean
+            5-minute bars.
+        relogin_every: Proactively refresh the baostock session every N symbols.
+
+    Yields:
+        ``(symbol, df)`` where ``df`` has raw minute bars or ``None`` when no
+        data is available.
+    """
+    query_frequency, frequency_label = _normalize_minute_frequency(frequency)
+    adjustflag = _BAOSTOCK_ADJUST["none"]
+
+    def _relogin() -> None:
+        try:
+            bs.logout()
+        except Exception:
+            pass
+        bs.login()
+
+    def _fetch(symbol: str) -> Optional[pd.DataFrame]:
+        """One Baostock minute query; returns df, None, or raises _SessionLost."""
+        code = f"{symbol[:2]}.{symbol[2:]}"
+        rs = bs.query_history_k_data_plus(
+            code=code,
+            fields=_MINUTE_FIELDS,
+            start_date=start,
+            end_date=end,
+            frequency=query_frequency,
+            adjustflag=adjustflag,
+        )
+        if rs.error_code != "0":
+            if "未登录" in (rs.error_msg or ""):
+                raise _SessionLost(rs.error_msg)
+            logger.warning("baostock minute error for %s: %s", symbol, rs.error_msg)
+            return None
+        rows = []
+        while rs.next():
+            rows.append(rs.get_row_data())
+        if not rows:
+            return None
+
+        df = pd.DataFrame(rows, columns=_MINUTE_FIELDS.split(","))
+        df[_MINUTE_NUMERIC] = df[_MINUTE_NUMERIC].apply(pd.to_numeric, errors="coerce")
+        df["datetime"] = _parse_minute_datetime(df["date"], df["time"])
+        bad_timestamps = df["datetime"].isna()
+        if bad_timestamps.any():
+            raise ValueError(
+                f"Could not parse {int(bad_timestamps.sum())} minute timestamp(s)"
+            )
+        df["date"] = pd.to_datetime(df["date"], errors="coerce")
+        df["time"] = df["datetime"].dt.strftime("%H:%M:%S")
+        df["frequency"] = frequency_label
+        df["vwap"] = (df["amount"] / df["volume"]).where(df["volume"] > 0)
+        df["symbol"] = symbol
+        return df[_MINUTE_COLUMNS].sort_values("datetime").reset_index(drop=True)
+
+    bs.login()
+    try:
+        for i, symbol in enumerate(symbols):
+            if i and relogin_every and i % relogin_every == 0:
+                _relogin()
+
+            df: Optional[pd.DataFrame] = None
+            for attempt in (1, 2):
+                try:
+                    df = _fetch(symbol)
+                    break
+                except _SessionLost:
+                    if attempt == 1:
+                        _relogin()
+                        continue
+                    logger.warning("baostock minute session lost for %s after relogin", symbol)
+                except Exception as e:
+                    logger.warning("baostock minute download failed for %s: %s", symbol, e)
+                    break
+
+            if df is not None and not df.empty:
+                yield symbol, df
+            else:
+                yield symbol, None
+    finally:
+        try:
+            bs.logout()
+        except Exception:
+            pass