diff --git a/download_models.py b/download_models.py new file mode 100644 index 000000000..b143bbfae --- /dev/null +++ b/download_models.py @@ -0,0 +1,144 @@ +""" +Kronos Model Download Script +Downloads tokenizer and Kronos-small model from HuggingFace Hub. + +Exit codes: + 0 -- download + smoke test succeeded + 1 -- failure (missing submodule, import error, or download error) +""" +import argparse +import sys +from pathlib import Path + +KRONOS_DIR = Path(__file__).resolve().parent +MODEL_DIR = KRONOS_DIR / "model" + +DEFAULT_TOKENIZER = "NeoQuasar/Kronos-Tokenizer-base" +DEFAULT_MODEL = "NeoQuasar/Kronos-small" + + +def _check_model_package() -> None: + if not MODEL_DIR.is_dir(): + print("[ERROR] Kronos model package not found at", MODEL_DIR) + print("Hint: Ensure the Kronos submodule is initialized:") + print(" git submodule update --init Kronos") + sys.exit(1) + + +def _import_model(): + """Import Kronos/KronosTokenizer with a human-readable failure path.""" + _check_model_package() + if str(KRONOS_DIR) not in sys.path: + sys.path.insert(0, str(KRONOS_DIR)) + try: + from model import Kronos, KronosTokenizer + + return Kronos, KronosTokenizer + except ImportError as e: + print(f"[ERROR] Failed to import Kronos model: {e}") + print("Hint: Ensure dependencies are installed " + "(pip install -r Kronos/requirements.txt) and the Kronos") + print(" submodule is initialized (git submodule update --init).") + sys.exit(1) + + +def _device_label(obj) -> str: + """Best-effort device string. Works for nn.Module, degrades gracefully.""" + if hasattr(obj, "parameters"): + try: + return str(next(obj.parameters()).device) + except (StopIteration, AttributeError): + return "unknown" + return "N/A (non-torch backend)" + + +def _smoke_test(model, tokenizer) -> bool: + """ + Verify the downloaded artifacts load and produce a forward pass. + + Kronos is a time-series foundation model: the tokenizer consumes a numeric + tensor and Kronos consumes token-id tensors, so the smoke test uses + synthetic tensors rather than text. + """ + print("\n[Verify] Running smoke test...") + try: + import torch + + # Tokenizer: forward pass on a synthetic time-series batch. + d_in = tokenizer.embed.in_features + x = torch.randn(1, 64, d_in) + with torch.no_grad(): + tokenizer(x) + print(" [OK] Tokenizer forward pass succeeded") + + # Model: forward pass on random token ids. + s1_vocab = getattr(model, "s1_vocab_size", 4096) + s2_vocab = getattr(model, "s2_vocab_size", 4096) + s1_ids = torch.randint(0, s1_vocab, (1, 64)) + s2_ids = torch.randint(0, s2_vocab, (1, 64)) + with torch.no_grad(): + model(s1_ids, s2_ids) + print(" [OK] Model forward pass succeeded") + return True + except Exception as e: + print(f" [FAIL] Smoke test failed: {e}") + return False + + +def download_models( + tokenizer_name: str = DEFAULT_TOKENIZER, + model_name: str = DEFAULT_MODEL, +) -> bool: + """ + Download the tokenizer and model, then verify them with a smoke test. + + Returns True on success; on failure prints an error and returns False. + """ + print("=" * 50) + print("Kronos Model Downloader") + print("=" * 50) + + model_cls, tokenizer_cls = _import_model() + + print("\n[1/2] Downloading KronosTokenizer...") + print(f" Model: {tokenizer_name}") + try: + tokenizer = tokenizer_cls.from_pretrained(tokenizer_name) + except Exception as e: + print(f"[ERROR] Tokenizer download failed: {e}") + return False + print(" [OK] Tokenizer downloaded successfully") + + print("\n[2/2] Downloading Kronos model...") + print(f" Model: {model_name}") + try: + model = model_cls.from_pretrained(model_name) + except Exception as e: + print(f"[ERROR] Model download failed: {e}") + return False + print(" [OK] Model downloaded successfully") + + ok = _smoke_test(model, tokenizer) + + print("\n" + "=" * 50) + if ok: + print("Download complete!") + else: + print("Download complete, but the smoke test failed -- the cached") + print("artifacts may be corrupt. Re-run to re-download them.") + print(f"Model device: {_device_label(model)}") + print(f"Tokenizer device: {_device_label(tokenizer)}") + print("=" * 50) + return ok + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Download Kronos models from HuggingFace Hub" + ) + parser.add_argument("--tokenizer", default=DEFAULT_TOKENIZER, help="HF tokenizer repo id") + parser.add_argument("--model", default=DEFAULT_MODEL, help="HF model repo id") + args = parser.parse_args() + + success = download_models(args.tokenizer, args.model) + sys.exit(0 if success else 1) diff --git a/examples/prediction_example.py b/examples/prediction_example.py index 880f22b86..d91dd5693 100644 --- a/examples/prediction_example.py +++ b/examples/prediction_example.py @@ -1,80 +1,141 @@ -import pandas as pd -import matplotlib.pyplot as plt -import sys -sys.path.append("../") -from model import Kronos, KronosTokenizer, KronosPredictor - - -def plot_prediction(kline_df, pred_df): - pred_df.index = kline_df.index[-pred_df.shape[0]:] - sr_close = kline_df['close'] - sr_pred_close = pred_df['close'] - sr_close.name = 'Ground Truth' - sr_pred_close.name = "Prediction" - - sr_volume = kline_df['volume'] - sr_pred_volume = pred_df['volume'] - sr_volume.name = 'Ground Truth' - sr_pred_volume.name = "Prediction" - - close_df = pd.concat([sr_close, sr_pred_close], axis=1) - volume_df = pd.concat([sr_volume, sr_pred_volume], axis=1) - - fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 6), sharex=True) - - ax1.plot(close_df['Ground Truth'], label='Ground Truth', color='blue', linewidth=1.5) - ax1.plot(close_df['Prediction'], label='Prediction', color='red', linewidth=1.5) - ax1.set_ylabel('Close Price', fontsize=14) - ax1.legend(loc='lower left', fontsize=12) - ax1.grid(True) - - ax2.plot(volume_df['Ground Truth'], label='Ground Truth', color='blue', linewidth=1.5) - ax2.plot(volume_df['Prediction'], label='Prediction', color='red', linewidth=1.5) - ax2.set_ylabel('Volume', fontsize=14) - ax2.legend(loc='upper left', fontsize=12) - ax2.grid(True) - - plt.tight_layout() - plt.show() - - -# 1. Load Model and Tokenizer -tokenizer = KronosTokenizer.from_pretrained("NeoQuasar/Kronos-Tokenizer-base") -model = Kronos.from_pretrained("NeoQuasar/Kronos-small") - -# 2. Instantiate Predictor -predictor = KronosPredictor(model, tokenizer, max_context=512) - -# 3. Prepare Data -df = pd.read_csv("./data/XSHG_5min_600977.csv") -df['timestamps'] = pd.to_datetime(df['timestamps']) - -lookback = 400 -pred_len = 120 - -x_df = df.loc[:lookback-1, ['open', 'high', 'low', 'close', 'volume', 'amount']] -x_timestamp = df.loc[:lookback-1, 'timestamps'] -y_timestamp = df.loc[lookback:lookback+pred_len-1, 'timestamps'] - -# 4. Make Prediction -pred_df = predictor.predict( - df=x_df, - x_timestamp=x_timestamp, - y_timestamp=y_timestamp, - pred_len=pred_len, - T=1.0, - top_p=0.9, - sample_count=1, - verbose=True -) - -# 5. Visualize Results -print("Forecasted Data Head:") -print(pred_df.head()) - -# Combine historical and forecasted data for plotting -kline_df = df.loc[:lookback+pred_len-1] - -# visualize -plot_prediction(kline_df, pred_df) - +import argparse +import sys +from pathlib import Path + +import pandas as pd + +# Anchor the project root on this file's location so the example can be run +# from any working directory (e.g. `python examples/prediction_example.py`). +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from model import Kronos, KronosPredictor, KronosTokenizer + +REQUIRED_COLUMNS = ['open', 'high', 'low', 'close', 'volume', 'amount', 'timestamps'] + + +def plot_prediction(kline_df, pred_df): + import matplotlib.pyplot as plt + + pred_df.index = kline_df.index[-pred_df.shape[0]:] + sr_close = kline_df['close'] + sr_pred_close = pred_df['close'] + sr_close.name = 'Ground Truth' + sr_pred_close.name = "Prediction" + + sr_volume = kline_df['volume'] + sr_pred_volume = pred_df['volume'] + sr_volume.name = 'Ground Truth' + sr_pred_volume.name = "Prediction" + + close_df = pd.concat([sr_close, sr_pred_close], axis=1) + volume_df = pd.concat([sr_volume, sr_pred_volume], axis=1) + + fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 6), sharex=True) + + ax1.plot(close_df['Ground Truth'], label='Ground Truth', color='blue', linewidth=1.5) + ax1.plot(close_df['Prediction'], label='Prediction', color='red', linewidth=1.5) + ax1.set_ylabel('Close Price', fontsize=14) + ax1.legend(loc='lower left', fontsize=12) + ax1.grid(True) + + ax2.plot(volume_df['Ground Truth'], label='Ground Truth', color='blue', linewidth=1.5) + ax2.plot(volume_df['Prediction'], label='Prediction', color='red', linewidth=1.5) + ax2.set_ylabel('Volume', fontsize=14) + ax2.legend(loc='upper left', fontsize=12) + ax2.grid(True) + + plt.tight_layout() + plt.show() + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Run Kronos inference on an OHLCV CSV file.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("csv_path", type=Path, + help="Path to a CSV with columns: " + ", ".join(REQUIRED_COLUMNS)) + parser.add_argument("--lookback", type=int, default=400, + help="Number of historical bars to condition on.") + parser.add_argument("--pred_len", type=int, default=120, + help="Number of bars to forecast.") + parser.add_argument("--max_context", type=int, default=512, + help="Maximum context length fed to the model.") + parser.add_argument("--device", type=str, default=None, + help="Inference device (defaults to KRONOS_DEVICE, then auto-detect).") + parser.add_argument("--top_k", type=int, default=0, + help="Top-k sampling threshold (0 disables).") + parser.add_argument("--top_p", type=float, default=0.9, + help="Nucleus sampling threshold.") + parser.add_argument("--T", type=float, default=1.0, + help="Sampling temperature.") + parser.add_argument("--sample_count", type=int, default=1, + help="Parallel samples per series (averaged).") + parser.add_argument("--no-show", action="store_true", + help="Skip the matplotlib plot.") + return parser.parse_args() + + +def main(): + args = parse_args() + + if not args.csv_path.is_file(): + raise FileNotFoundError(f"CSV file not found: {args.csv_path}") + + df = pd.read_csv(args.csv_path) + missing = [c for c in REQUIRED_COLUMNS if c not in df.columns] + if missing: + raise ValueError( + f"CSV at {args.csv_path} is missing required columns: {missing}. " + f"Expected columns: {REQUIRED_COLUMNS}" + ) + + df['timestamps'] = pd.to_datetime(df['timestamps']) + + if args.lookback < 1 or args.pred_len < 1: + raise ValueError("lookback and pred_len must both be >= 1.") + if args.lookback + args.pred_len > len(df): + raise ValueError( + f"lookback + pred_len ({args.lookback + args.pred_len}) exceeds the " + f"number of rows in the CSV ({len(df)})." + ) + + # 1. Load Model and Tokenizer + tokenizer = KronosTokenizer.from_pretrained("NeoQuasar/Kronos-Tokenizer-base") + model = Kronos.from_pretrained("NeoQuasar/Kronos-small") + + # 2. Instantiate Predictor + predictor = KronosPredictor(model, tokenizer, device=args.device, max_context=args.max_context) + + # 3. Prepare Data + lookback = args.lookback + pred_len = args.pred_len + + x_df = df.loc[:lookback - 1, ['open', 'high', 'low', 'close', 'volume', 'amount']] + x_timestamp = df.loc[:lookback - 1, 'timestamps'] + y_timestamp = df.loc[lookback:lookback + pred_len - 1, 'timestamps'] + + # 4. Make Prediction + pred_df = predictor.predict( + df=x_df, + x_timestamp=x_timestamp, + y_timestamp=y_timestamp, + pred_len=pred_len, + T=args.T, + top_k=args.top_k, + top_p=args.top_p, + sample_count=args.sample_count, + verbose=True + ) + + # 5. Visualize Results + print("Forecasted Data Head:") + print(pred_df.head()) + + if not args.no_show: + # Combine historical and forecasted data for plotting + kline_df = df.loc[:lookback + pred_len - 1] + plot_prediction(kline_df, pred_df) + + +if __name__ == '__main__': + main() diff --git a/finetune/config.py b/finetune/config.py index 04cc3ee31..dc262bdca 100644 --- a/finetune/config.py +++ b/finetune/config.py @@ -1,5 +1,6 @@ import os + class Config: """ Configuration class for the entire project. @@ -51,8 +52,8 @@ def __init__(self): # Number of samples to draw for one "epoch" of training/validation. # This is useful for large datasets where a true epoch is too long. - self.n_train_iter = 2000 * self.batch_size - self.n_val_iter = 400 * self.batch_size + self.n_train_samples_per_epoch = 2000 * self.batch_size + self.n_val_samples_per_epoch = 400 * self.batch_size # Learning rates for different model components. self.tokenizer_learning_rate = 2e-4 @@ -74,11 +75,12 @@ def __init__(self): # ================================================================= self.use_comet = True # Set to False if you don't want to use Comet ML self.comet_config = { - # It is highly recommended to load secrets from environment variables - # for security purposes. Example: os.getenv("COMET_API_KEY") - "api_key": "YOUR_COMET_API_KEY", + # Secrets are loaded from environment variables so they are never + # committed to the repository. Comet is skipped entirely when the + # API key is missing. + "api_key": os.getenv("COMET_API_KEY", ""), "project_name": "Kronos-Finetune-Demo", - "workspace": "your_comet_workspace" # TODO: Change to your Comet ML workspace name + "workspace": os.getenv("COMET_WORKSPACE", "your_comet_workspace") # TODO: Change to your Comet ML workspace name } self.comet_tag = 'finetune_demo' self.comet_name = 'finetune_demo' @@ -117,8 +119,36 @@ def __init__(self): self.inference_top_k = 0 self.inference_sample_count = 5 self.backtest_batch_size = 1000 + self.inference_num_workers = 4 # Upper bound for inference DataLoader workers. self.backtest_benchmark = self._set_benchmark(self.instrument) + # Market profile used by the backtest exchange simulator. This is what + # turns the "general" framework into a specific market: + # 'cn' - Chinese A-share: T+1, daily open dealing, ~10/15bp costs, 9.5% limit-up/down. + # 'generic' - Neutral profile: no price limits, close dealing, negligible costs + # (suited to forex/crypto/tick-style evaluation). + self.market_profile = 'cn' + self.backtest_freq = "day" + self.backtest_time_per_step = "day" + self.backtest_account = 100_000_000 + self.backtest_exchange_kwargs = self._set_exchange_kwargs(self.market_profile, self.backtest_freq) + # Qlib provider region used by qlib.init(): 'cn' or 'us'. + self.qlib_region = 'cn' + + def _set_exchange_kwargs(self, market_profile: str, freq: str) -> dict: + if market_profile == 'cn': + return { + "freq": freq, "limit_threshold": 0.095, "deal_price": "open", + "open_cost": 0.001, "close_cost": 0.0015, "min_cost": 5, + } + elif market_profile == 'generic': + return { + "freq": freq, "limit_threshold": None, "deal_price": "close", + "open_cost": 0.0001, "close_cost": 0.0001, "min_cost": 0, + } + else: + raise ValueError(f"Unknown market_profile: {market_profile}") + def _set_benchmark(self, instrument): dt_benchmark = { 'csi800': "SH000906", diff --git a/finetune/dataset.py b/finetune/dataset.py index ae4f7242b..bc052ce1a 100644 --- a/finetune/dataset.py +++ b/finetune/dataset.py @@ -1,9 +1,10 @@ import pickle import random + import numpy as np import torch -from torch.utils.data import Dataset from config import Config +from torch.utils.data import Dataset class QlibDataset(Dataset): @@ -33,10 +34,10 @@ def __init__(self, data_type: str = 'train'): # Set paths and number of samples based on the data type. if data_type == 'train': self.data_path = f"{self.config.dataset_path}/train_data.pkl" - self.n_samples = self.config.n_train_iter + self.n_samples = self.config.n_train_samples_per_epoch else: self.data_path = f"{self.config.dataset_path}/val_data.pkl" - self.n_samples = self.config.n_val_iter + self.n_samples = self.config.n_val_samples_per_epoch with open(self.data_path, 'rb') as f: self.data = pickle.load(f) @@ -90,7 +91,7 @@ def __len__(self) -> int: return self.n_samples def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]: - + # Select a random sample from the entire pool of indices. random_idx = self.py_rng.randint(0, len(self.indices) - 1) symbol, start_idx = self.indices[random_idx] @@ -112,8 +113,13 @@ def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]: x_mean = np.mean(past_x, axis=0) x_std = np.std(past_x, axis=0) + # Floor the std so flat lookback windows cannot amplify noise via a + # near-zero denominator. The small additive epsilon is retained to match + # pretrained normalization numerics; it is irrelevant once floored. + x_std = np.maximum(x_std + 1e-5, 1e-3) + # Apply normalization and robust clipping to the entire sequence - x = (x - x_mean) / (x_std + 1e-5) + x = (x - x_mean) / x_std x = np.clip(x, -self.config.clip, self.config.clip) # Convert to PyTorch tensors. diff --git a/finetune/qlib_test.py b/finetune/qlib_test.py index 97aebe419..6a3614b06 100644 --- a/finetune/qlib_test.py +++ b/finetune/qlib_test.py @@ -1,29 +1,31 @@ -import os -import sys import argparse +import os import pickle +import sys from collections import defaultdict +from pathlib import Path import numpy as np import pandas as pd +import qlib import torch -from torch.utils.data import Dataset, DataLoader -from tqdm import trange, tqdm from matplotlib import pyplot as plt - -import qlib -from qlib.config import REG_CN -from qlib.backtest import backtest, executor, CommonInfrastructure +from qlib.backtest import backtest, executor +from qlib.config import REG_CN, REG_US from qlib.contrib.evaluate import risk_analysis from qlib.contrib.strategy import TopkDropoutStrategy -from qlib.utils import flatten_dict from qlib.utils.time import Freq +from torch.utils.data import DataLoader, Dataset +from tqdm import tqdm -# Ensure project root is in the Python path -sys.path.append("../") +# Anchor the project root on this file's location so the script can be launched +# from any working directory (e.g. `python finetune/qlib_test.py`). +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from config import Config from model.kronos import Kronos, KronosTokenizer, auto_regressive_inference +_QLIB_REGIONS = {"cn": REG_CN, "us": REG_US} + # ================================================================================= # 1. Data Loading and Processing for Inference @@ -46,30 +48,37 @@ def __init__(self, data: dict, config: Config): self.feature_list = config.feature_list self.time_feature_list = config.time_feature_list self.indices = [] + # Cache of processed (feature columns + time features) frames, computed + # lazily per symbol on first access instead of for every symbol at init. + self._processed = {} print("Preprocessing and building indices for test dataset...") for symbol in self.symbols: df = self.data[symbol].reset_index() - # Generate time features on-the-fly - df['minute'] = df['datetime'].dt.minute - df['hour'] = df['datetime'].dt.hour - df['weekday'] = df['datetime'].dt.weekday - df['day'] = df['datetime'].dt.day - df['month'] = df['datetime'].dt.month - self.data[symbol] = df # Store preprocessed dataframe - num_samples = len(df) - self.window_size + 1 if num_samples > 0: for i in range(num_samples): timestamp = df.iloc[i + self.config.lookback_window - 1]['datetime'] self.indices.append((symbol, i, timestamp)) + def _get_processed(self, symbol: str) -> pd.DataFrame: + """Returns the feature + time-feature frame for a symbol, computing it once.""" + if symbol not in self._processed: + df = self.data[symbol].reset_index() + df['minute'] = df['datetime'].dt.minute + df['hour'] = df['datetime'].dt.hour + df['weekday'] = df['datetime'].dt.weekday + df['day'] = df['datetime'].dt.day + df['month'] = df['datetime'].dt.month + self._processed[symbol] = df + return self._processed[symbol] + def __len__(self) -> int: return len(self.indices) def __getitem__(self, idx: int): symbol, start_idx, timestamp = self.indices[idx] - df = self.data[symbol] + df = self._get_processed(symbol) context_end = start_idx + self.config.lookback_window predict_end = context_end + self.config.predict_window @@ -83,7 +92,8 @@ def __getitem__(self, idx: int): # Instance-level normalization, consistent with training x_mean, x_std = np.mean(x, axis=0), np.std(x, axis=0) - x = (x - x_mean) / (x_std + 1e-5) + x_std = np.maximum(x_std + 1e-5, 1e-3) # Floor near-zero denominators on flat windows + x = (x - x_mean) / x_std x = np.clip(x, -self.config.clip, self.config.clip) return torch.from_numpy(x), torch.from_numpy(x_stamp), torch.from_numpy(y_stamp), symbol, timestamp @@ -105,7 +115,10 @@ def __init__(self, config: Config): def initialize_qlib(self): """Initializes the Qlib environment.""" print("Initializing Qlib for backtesting...") - qlib.init(provider_uri=self.config.qlib_data_path, region=REG_CN) + region = _QLIB_REGIONS.get(self.config.qlib_region) + if region is None: + raise ValueError(f"Unsupported qlib_region: {self.config.qlib_region}") + qlib.init(provider_uri=self.config.qlib_data_path, region=region) def run_single_backtest(self, signal_series: pd.Series) -> pd.DataFrame: """ @@ -124,24 +137,21 @@ def run_single_backtest(self, signal_series: pd.Series) -> pd.DataFrame: signal=signal_series, ) executor_config = { - "time_per_step": "day", + "time_per_step": self.config.backtest_time_per_step, "generate_portfolio_metrics": True, "delay_execution": True, } backtest_config = { "start_time": self.config.backtest_time_range[0], "end_time": self.config.backtest_time_range[1], - "account": 100_000_000, + "account": self.config.backtest_account, "benchmark": self.config.backtest_benchmark, - "exchange_kwargs": { - "freq": "day", "limit_threshold": 0.095, "deal_price": "open", - "open_cost": 0.001, "close_cost": 0.0015, "min_cost": 5, - }, + "exchange_kwargs": dict(self.config.backtest_exchange_kwargs), "executor": executor.SimulatorExecutor(**executor_config), } portfolio_metric_dict, _ = backtest(strategy=strategy, **backtest_config) - analysis_freq = "{0}{1}".format(*Freq.parse("day")) + analysis_freq = "{}{}".format(*Freq.parse(self.config.backtest_freq)) report, _ = portfolio_metric_dict.get(analysis_freq) # --- Analysis and Reporting --- @@ -196,7 +206,9 @@ def run_and_plot_results(self, signals: dict[str, pd.DataFrame]): axes[1].set_ylabel("Cumulative Excess Return") plt.tight_layout() - plt.savefig("../figures/backtest_result_example.png", dpi=200) + figures_dir = Path(__file__).resolve().parent.parent / "figures" + figures_dir.mkdir(parents=True, exist_ok=True) + plt.savefig(str(figures_dir / "backtest_result_example.png"), dpi=200) plt.show() @@ -225,7 +237,7 @@ def collate_fn_for_inference(batch): A single tuple containing the batched data. """ # Unzip the list of samples into separate lists for each data type - x, x_stamp, y_stamp, symbols, timestamps = zip(*batch) + x, x_stamp, y_stamp, symbols, timestamps = zip(*batch, strict=True) # Stack the tensors to create a batch x_batch = torch.stack(x, dim=0) @@ -236,13 +248,16 @@ def collate_fn_for_inference(batch): return x_batch, x_stamp_batch, y_stamp_batch, list(symbols), list(timestamps) -def generate_predictions(config: dict, test_data: dict) -> dict[str, pd.DataFrame]: +def generate_predictions(config: dict, test_data: dict, base_config: Config = None) -> dict[str, pd.DataFrame]: """ Runs inference on the test dataset to generate prediction signals. Args: config (dict): A dictionary containing inference parameters. test_data (dict): The raw test data loaded from a pickle file. + base_config (Config, optional): The base Config instance whose data + schema (feature/time feature lists, window sizes) is used to build + the inference dataset. Falls back to a fresh Config() if omitted. Returns: A dictionary where keys are signal types (e.g., 'mean', 'last') and @@ -251,13 +266,14 @@ def generate_predictions(config: dict, test_data: dict) -> dict[str, pd.DataFram tokenizer, model = load_models(config) device = torch.device(config['device']) - # Use the Dataset and DataLoader for efficient batching and processing - dataset = QlibTestDataset(data=test_data, config=Config()) + # Use the caller's base_config so user overrides (feature_list, windows, + # clipping, ...) are respected instead of silently constructing defaults. + dataset = QlibTestDataset(data=test_data, config=base_config if base_config is not None else Config()) loader = DataLoader( dataset, batch_size=config['batch_size'] // config['sample_count'], shuffle=False, - num_workers=os.cpu_count() // 2, + num_workers=min(config.get('num_workers', 4), 8), collate_fn=collate_fn_for_inference ) @@ -324,6 +340,7 @@ def main(): 'top_p': base_config.inference_top_p, 'sample_count': base_config.inference_sample_count, 'batch_size': base_config.backtest_batch_size, + 'num_workers': base_config.inference_num_workers, } print("--- Running with Configuration ---") @@ -338,7 +355,7 @@ def main(): test_data = pickle.load(f) print(test_data) # --- 3. Generate Predictions --- - model_preds = generate_predictions(run_config, test_data) + model_preds = generate_predictions(run_config, test_data, base_config) # --- 4. Save Predictions --- save_dir = os.path.join(run_config['result_save_path'], run_config['result_name']) diff --git a/finetune/train_predictor.py b/finetune/train_predictor.py index 47eddc91f..d78c781b6 100644 --- a/finetune/train_predictor.py +++ b/finetune/train_predictor.py @@ -1,29 +1,26 @@ +import json import os import sys -import json import time +from pathlib import Path from time import gmtime, strftime -import torch.distributed as dist + +import comet_ml import torch +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP # noqa: N817 from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler -from torch.nn.parallel import DistributedDataParallel as DDP - -import comet_ml -# Ensure project root is in path -sys.path.append('../') +# Anchor the project root on this file's location so the script can be launched +# from any working directory (e.g. `python finetune/train_predictor.py`). +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from config import Config from dataset import QlibDataset -from model.kronos import KronosTokenizer, Kronos +from model.kronos import Kronos, KronosTokenizer + # Import shared utilities -from utils.training_utils import ( - setup_ddp, - cleanup_ddp, - set_seed, - get_model_size, - format_time -) +from utils.training_utils import cleanup_ddp, format_time, get_model_size, set_seed, setup_ddp def create_dataloaders(config: dict, rank: int, world_size: int): @@ -104,9 +101,11 @@ def train_model(model, tokenizer, device, config, save_dir, logger, rank, world_ token_in = [token_seq_0[:, :-1], token_seq_1[:, :-1]] token_out = [token_seq_0[:, 1:], token_seq_1[:, 1:]] - # Forward pass and loss calculation + # Forward pass and loss calculation. compute_loss is called through + # the (possibly DDP-wrapped) module so gradients stay inside the DDP + # wrapper instead of poking into model.module directly. logits = model(token_in[0], token_in[1], batch_x_stamp[:, :-1, :]) - loss, s1_loss, s2_loss = model.module.head.compute_loss(logits[0], logits[1], token_out[0], token_out[1]) + loss, s1_loss, s2_loss = model.compute_loss(logits[0], logits[1], token_out[0], token_out[1]) # Backward pass and optimization optimizer.zero_grad() @@ -135,17 +134,23 @@ def train_model(model, tokenizer, device, config, save_dir, logger, rank, world_ model.eval() tot_val_loss_sum_rank = 0.0 val_batches_processed_rank = 0 - with torch.no_grad(): - for batch_x, batch_x_stamp in val_loader: - batch_x = batch_x.to(device, non_blocking=True) - batch_x_stamp = batch_x_stamp.to(device, non_blocking=True) + with torch.no_grad(): + for batch_x, batch_x_stamp in val_loader: + batch_x = batch_x.to(device, non_blocking=True) + batch_x_stamp = batch_x_stamp.to(device, non_blocking=True) token_seq_0, token_seq_1 = tokenizer.encode(batch_x, half=True) token_in = [token_seq_0[:, :-1], token_seq_1[:, :-1]] token_out = [token_seq_0[:, 1:], token_seq_1[:, 1:]] - logits = model(token_in[0], token_in[1], batch_x_stamp[:, :-1, :]) - val_loss, _, _ = model.module.head.compute_loss(logits[0], logits[1], token_out[0], token_out[1]) + # Teacher-forcing during validation makes the reported loss + # deterministic (the s1 path is otherwise stochastic via + # torch.multinomial sampling in the forward pass). + logits = model( + token_in[0], token_in[1], batch_x_stamp[:, :-1, :], + use_teacher_forcing=True, s1_targets=token_out[0] + ) + val_loss, _, _ = model.compute_loss(logits[0], logits[1], token_out[0], token_out[1]) tot_val_loss_sum_rank += val_loss.item() val_batches_processed_rank += 1 @@ -196,7 +201,7 @@ def main(config: dict): 'save_directory': save_dir, 'world_size': world_size, } - if config['use_comet']: + if config['use_comet'] and config['comet_config'].get('api_key'): comet_logger = comet_ml.Experiment( api_key=config['comet_config']['api_key'], project_name=config['comet_config']['project_name'], @@ -206,10 +211,15 @@ def main(config: dict): comet_logger.set_name(config['comet_name']) comet_logger.log_parameters(config) print("Comet Logger Initialized.") + elif rank == 0: + print("Comet logging disabled (use_comet=False or no COMET_API_KEY set).") dist.barrier() # Model Initialization + # NOTE: The tokenizer stays frozen and is only ever used under torch.no_grad() + # in train_model(). If it is ever unfrozen for end-to-end fine-tuning, the + # dropout inside its encoder would corrupt the produced token IDs. tokenizer = KronosTokenizer.from_pretrained(config['finetuned_tokenizer_path']) tokenizer.eval().to(device) @@ -230,7 +240,8 @@ def main(config: dict): with open(os.path.join(save_dir, 'summary.json'), 'w') as f: json.dump(master_summary, f, indent=4) print('Training finished. Summary file saved.') - if comet_logger: comet_logger.end() + if comet_logger: + comet_logger.end() cleanup_ddp() diff --git a/model/kronos.py b/model/kronos.py index ce4494ee0..da0edad6c 100644 --- a/model/kronos.py +++ b/model/kronos.py @@ -1,13 +1,22 @@ +import os + import numpy as np import pandas as pd import torch +import torch.nn as nn +import torch.nn.functional as F # noqa: N812 from huggingface_hub import PyTorchModelHubMixin -import sys - from tqdm import trange -sys.path.append("../") -from model.module import * +from .module import ( + BSQuantizer, + DependencyAwareLayer, + DualHead, + HierarchicalEmbedding, + RMSNorm, + TemporalEmbedding, + TransformerBlock, +) class KronosTokenizer(nn.Module, PyTorchModelHubMixin): @@ -57,11 +66,16 @@ def __init__(self, d_in, d_model, n_heads, ff_dim, n_enc_layers, n_dec_layers, f self.head = nn.Linear(self.d_model, self.d_in) # Encoder Transformer Blocks + # NOTE: n_enc_layers - 1 TransformerBlocks are instantiated. The linear + # input projection (self.embed) plays the role of the "first" transform + # and the quant embedding (self.quant_embed) that of the "last", so the + # effective stack depth matches the paper's reported architecture. self.encoder = nn.ModuleList([ TransformerBlock(self.d_model, self.n_heads, self.ff_dim, self.ffn_dropout_p, self.attn_dropout_p, self.resid_dropout_p) for _ in range(self.enc_layers - 1) ]) - # Decoder Transformer Blocks + # Decoder Transformer Blocks (same reasoning as encoder; the + # post_quant_embed projection and self.head bookend the block stack). self.decoder = nn.ModuleList([ TransformerBlock(self.d_model, self.n_heads, self.ff_dim, self.ffn_dropout_p, self.attn_dropout_p, self.resid_dropout_p) for _ in range(self.dec_layers - 1) @@ -85,6 +99,14 @@ def forward(self, x): - torch.Tensor: bsq_loss - Loss from the BSQuantizer. - torch.Tensor: quantized - Quantized representation from BSQuantizer. - torch.Tensor: z_indices - Indices from the BSQuantizer. + + Note: + self.decoder is a single ModuleList that is applied twice in one + forward pass (once for the coarse s1 reconstruction, once for the + full codebook). This is intentional weight sharing: both + reconstruction objectives back-propagate into the same decoder + weights. Because TransformerBlock uses RMSNorm, no running + statistics are corrupted by the double pass. """ z = self.embed(x) @@ -158,13 +180,15 @@ def encode(self, x, half=False): bsq_loss, quantized, z_indices = self.tokenizer(z, half=half, collect_metrics=False) return z_indices - def decode(self, x, half=False): + def decode(self, x, half=False, padding_mask=None): """ Decodes quantized indices back to the input data space. Args: x (torch.Tensor): Quantized indices tensor. half (bool, optional): Whether the indices were generated with half quantization. Defaults to False. + padding_mask (torch.Tensor, optional): Mask of shape [batch_size, seq_len] where 1 marks + padding positions that attention must ignore. Defaults to None. Returns: torch.Tensor: Reconstructed output tensor of shape (batch_size, seq_len, d_in). @@ -172,7 +196,7 @@ def decode(self, x, half=False): quantized = self.indices_to_bits(x, half) z = self.post_quant_embed(quantized) for layer in self.decoder: - z = layer(z) + z = layer(z, key_padding_mask=padding_mask) z = self.head(z) return z @@ -327,6 +351,15 @@ def decode_s2(self, context, s1_ids, padding_mask=None): x2 = self.dep_layer(context, sibling_embed, key_padding_mask=padding_mask) return self.head.cond_forward(x2) + def compute_loss(self, s1_logits, s2_logits, s1_targets, s2_targets, padding_mask=None): + """Computes the combined S1+S2 cross-entropy loss. + + Exposed on the module itself (not just ``self.head``) so that + DistributedDataParallel-wrapped models can call it without poking + through ``model.module`` and bypassing the DDP wrapper. + """ + return self.head.compute_loss(s1_logits, s2_logits, s1_targets, s2_targets, padding_mask) + def top_k_top_p_filtering( logits, @@ -349,7 +382,6 @@ def top_k_top_p_filtering( # Remove all tokens with a probability less than the last token of the top-k indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] logits[indices_to_remove] = filter_value - return logits if top_p < 1.0: sorted_logits, sorted_indices = torch.sort(logits, descending=True) @@ -367,14 +399,14 @@ def top_k_top_p_filtering( # scatter sorted tensors to original indexing indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) logits[indices_to_remove] = filter_value - return logits + + return logits def sample_from_logits(logits, temperature=1.0, top_k=None, top_p=None, sample_logits=True): logits = logits / temperature - if top_k is not None or top_p is not None: - if top_k > 0 or top_p < 1.0: - logits = top_k_top_p_filtering(logits, top_k=top_k, top_p=top_p) + if (top_k is not None and top_k > 0) or (top_p is not None and top_p < 1.0): + logits = top_k_top_p_filtering(logits, top_k=top_k, top_p=top_p) probs = F.softmax(logits, dim=-1) @@ -386,17 +418,44 @@ def sample_from_logits(logits, temperature=1.0, top_k=None, top_p=None, sample_l return x -def auto_regressive_inference(tokenizer, model, x, x_stamp, y_stamp, max_context, pred_len, clip=5, T=1.0, top_k=0, top_p=0.99, sample_count=5, verbose=False): +def auto_regressive_inference(tokenizer, model, x, x_stamp, y_stamp, max_context, pred_len, clip=5, T=1.0, top_k=0, top_p=0.99, sample_count=5, verbose=False, padding_mask=None): # noqa: N803 + """ + Autoregressive inference over the Kronos token space. + + Args: + tokenizer (KronosTokenizer): Frozen tokenizer. + model (Kronos): Predictor model. + x (torch.Tensor): Normalized input of shape [batch_size, seq_len, d_in]. + x_stamp (torch.Tensor): Historical time features [batch_size, seq_len, time_feat]. + y_stamp (torch.Tensor): Future time features [batch_size, pred_len, time_feat]. + max_context (int): Maximum number of tokens fed to the transformer. + pred_len (int): Number of steps to generate. + clip (float): Value clipping bound for inputs. + T (float): Sampling temperature. + top_k (int): Top-k filtering threshold (applied together with top_p when both set). + top_p (float): Nucleus (top-p) filtering threshold. + sample_count (int): Number of parallel samples per series (averaged at the end). + verbose (bool): Whether to display a progress bar. + padding_mask (torch.Tensor, optional): Mask of shape [batch_size, seq_len] where 1 marks + left-padded positions that attention must ignore. Enables batching series of unequal length. + + Returns: + np.ndarray: Predictions of shape [batch_size, pred_len, d_in]. + """ with torch.no_grad(): x = torch.clip(x, -clip, clip) device = x.device + if padding_mask is not None: + padding_mask = (padding_mask != 0).to(device) x = x.unsqueeze(1).repeat(1, sample_count, 1, 1).reshape(-1, x.size(1), x.size(2)).to(device) x_stamp = x_stamp.unsqueeze(1).repeat(1, sample_count, 1, 1).reshape(-1, x_stamp.size(1), x_stamp.size(2)).to(device) y_stamp = y_stamp.unsqueeze(1).repeat(1, sample_count, 1, 1).reshape(-1, y_stamp.size(1), y_stamp.size(2)).to(device) + if padding_mask is not None: + padding_mask = padding_mask.unsqueeze(1).repeat(1, sample_count, 1).reshape(-1, padding_mask.size(1)).to(device) x_token = tokenizer.encode(x, half=True) - + initial_seq_len = x.size(1) batch_size = x_token[0].size(0) total_seq_len = initial_seq_len + pred_len @@ -407,16 +466,18 @@ def auto_regressive_inference(tokenizer, model, x, x_stamp, y_stamp, max_context pre_buffer = x_token[0].new_zeros(batch_size, max_context) post_buffer = x_token[1].new_zeros(batch_size, max_context) + mask_buffer = None + if padding_mask is not None: + mask_buffer = padding_mask.new_zeros(batch_size, max_context, dtype=torch.bool) buffer_len = min(initial_seq_len, max_context) if buffer_len > 0: start_idx = max(0, initial_seq_len - max_context) pre_buffer[:, :buffer_len] = x_token[0][:, start_idx:start_idx + buffer_len] post_buffer[:, :buffer_len] = x_token[1][:, start_idx:start_idx + buffer_len] + if mask_buffer is not None: + mask_buffer[:, :buffer_len] = padding_mask[:, start_idx:start_idx + buffer_len] - if verbose: - ran = trange - else: - ran = range + ran = trange if verbose else range for i in ran(pred_len): current_seq_len = initial_seq_len + i window_len = min(current_seq_len, max_context) @@ -426,18 +487,20 @@ def auto_regressive_inference(tokenizer, model, x, x_stamp, y_stamp, max_context pre_buffer[:, :window_len], post_buffer[:, :window_len] ] + mask = mask_buffer[:, :window_len] if mask_buffer is not None else None else: input_tokens = [pre_buffer, post_buffer] + mask = mask_buffer if mask_buffer is not None else None context_end = current_seq_len context_start = max(0, context_end - max_context) current_stamp = full_stamp[:, context_start:context_end, :].contiguous() - s1_logits, context = model.decode_s1(input_tokens[0], input_tokens[1], current_stamp) + s1_logits, context = model.decode_s1(input_tokens[0], input_tokens[1], current_stamp, padding_mask=mask) s1_logits = s1_logits[:, -1, :] sample_pre = sample_from_logits(s1_logits, temperature=T, top_k=top_k, top_p=top_p, sample_logits=True) - s2_logits = model.decode_s2(context, sample_pre) + s2_logits = model.decode_s2(context, sample_pre, padding_mask=mask) s2_logits = s2_logits[:, -1, :] sample_post = sample_from_logits(s2_logits, temperature=T, top_k=top_k, top_p=top_p, sample_logits=True) @@ -447,11 +510,16 @@ def auto_regressive_inference(tokenizer, model, x, x_stamp, y_stamp, max_context if current_seq_len < max_context: pre_buffer[:, current_seq_len] = sample_pre.squeeze(-1) post_buffer[:, current_seq_len] = sample_post.squeeze(-1) + if mask_buffer is not None: + mask_buffer[:, current_seq_len] = False else: - pre_buffer.copy_(torch.roll(pre_buffer, shifts=-1, dims=1)) - post_buffer.copy_(torch.roll(post_buffer, shifts=-1, dims=1)) + pre_buffer[:, :-1] = pre_buffer[:, 1:].clone() + post_buffer[:, :-1] = post_buffer[:, 1:].clone() pre_buffer[:, -1] = sample_pre.squeeze(-1) post_buffer[:, -1] = sample_post.squeeze(-1) + if mask_buffer is not None: + mask_buffer[:, :-1] = mask_buffer[:, 1:].clone() + mask_buffer[:, -1] = False full_pre = torch.cat([x_token[0], generated_pre], dim=1) full_post = torch.cat([x_token[1], generated_post], dim=1) @@ -461,7 +529,14 @@ def auto_regressive_inference(tokenizer, model, x, x_stamp, y_stamp, max_context full_pre[:, context_start:total_seq_len].contiguous(), full_post[:, context_start:total_seq_len].contiguous() ] - z = tokenizer.decode(input_tokens, half=True) + if padding_mask is not None: + full_seq_mask = torch.cat( + [padding_mask, padding_mask.new_zeros(padding_mask.size(0), pred_len, dtype=torch.bool)], dim=1 + ) + decode_mask = full_seq_mask[:, context_start:total_seq_len].contiguous() + else: + decode_mask = None + z = tokenizer.decode(input_tokens, half=True, padding_mask=decode_mask) z = z.reshape(-1, sample_count, z.size(1), z.size(2)) preds = z.cpu().numpy() preds = np.mean(preds, axis=1) @@ -490,33 +565,53 @@ def __init__(self, model, tokenizer, device=None, max_context=512, clip=5): self.vol_col = 'volume' self.amt_vol = 'amount' self.time_cols = ['minute', 'hour', 'weekday', 'day', 'month'] - - # Auto-detect device if not specified + + # Auto-detect device if not specified. The KRONOS_DEVICE environment + # variable overrides auto-detection so operators can pin a specific GPU. if device is None: - if torch.cuda.is_available(): - device = "cuda:0" - elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): - device = "mps" - else: - device = "cpu" - + device = os.environ.get("KRONOS_DEVICE") + if device is None: + if torch.cuda.is_available(): + device = "cuda:0" + elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + device = "mps" + else: + device = "cpu" + elif isinstance(device, torch.device): + device = str(device) + self.device = device self.tokenizer = self.tokenizer.to(self.device) self.model = self.model.to(self.device) - def generate(self, x, x_stamp, y_stamp, pred_len, T, top_k, top_p, sample_count, verbose): + @staticmethod + def _floored_std(x_std, eps=1e-5, floor=1e-3): + """Floor the per-feature standard deviation to avoid blow-ups on flat windows. + + A flat lookback window yields x_std ~ 0; dividing by an arbitrarily small + epsilon (1e-5) alone amplifies any residual noise by up to 1e5x. The small + additive epsilon is kept (it is part of the pretrained normalization) and + a hard floor guarantees flat windows stay numerically stable. + """ + return np.maximum(x_std + eps, floor) + + def generate(self, x, x_stamp, y_stamp, pred_len, T, top_k, top_p, sample_count, verbose, padding_mask=None): # noqa: N803 x_tensor = torch.from_numpy(np.array(x).astype(np.float32)).to(self.device) x_stamp_tensor = torch.from_numpy(np.array(x_stamp).astype(np.float32)).to(self.device) y_stamp_tensor = torch.from_numpy(np.array(y_stamp).astype(np.float32)).to(self.device) + if padding_mask is not None: + padding_mask_tensor = torch.from_numpy(np.array(padding_mask)).to(self.device) + else: + padding_mask_tensor = None preds = auto_regressive_inference(self.tokenizer, self.model, x_tensor, x_stamp_tensor, y_stamp_tensor, self.max_context, pred_len, - self.clip, T, top_k, top_p, sample_count, verbose) + self.clip, T, top_k, top_p, sample_count, verbose, padding_mask_tensor) preds = preds[:, -pred_len:, :] return preds - def predict(self, df, x_timestamp, y_timestamp, pred_len, T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=True): + def predict(self, df, x_timestamp, y_timestamp, pred_len, T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=True): # noqa: N803 if not isinstance(df, pd.DataFrame): raise ValueError("Input must be a pandas DataFrame.") @@ -542,8 +637,9 @@ def predict(self, df, x_timestamp, y_timestamp, pred_len, T=1.0, top_k=0, top_p= y_stamp = y_time_df.values.astype(np.float32) x_mean, x_std = np.mean(x, axis=0), np.std(x, axis=0) + x_std = self._floored_std(x_std) - x = (x - x_mean) / (x_std + 1e-5) + x = (x - x_mean) / x_std x = np.clip(x, -self.clip, self.clip) x = x[np.newaxis, :] @@ -553,15 +649,20 @@ def predict(self, df, x_timestamp, y_timestamp, pred_len, T=1.0, top_k=0, top_p= preds = self.generate(x, x_stamp, y_stamp, pred_len, T, top_k, top_p, sample_count, verbose) preds = preds.squeeze(0) - preds = preds * (x_std + 1e-5) + x_mean + preds = preds * x_std + x_mean pred_df = pd.DataFrame(preds, columns=self.price_cols + [self.vol_col, self.amt_vol], index=y_timestamp) return pred_df - def predict_batch(self, df_list, x_timestamp_list, y_timestamp_list, pred_len, T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=True): + def predict_batch(self, df_list, x_timestamp_list, y_timestamp_list, pred_len, T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=True): # noqa: N803 """ - Perform parallel (batch) prediction on multiple time series. All series must have the same historical length and prediction length (pred_len). + Perform parallel (batch) prediction on multiple time series. + + Historical lengths may differ between series: inputs are left-padded to + the longest series and a padding mask is threaded through inference so + that the model never attends to padding. All series must share the same + prediction length (pred_len). Args: df_list (List[pd.DataFrame]): List of input DataFrames, each containing price columns and optional volume/amount columns. @@ -627,7 +728,8 @@ def predict_batch(self, df_list, x_timestamp_list, y_timestamp_list, pred_len, T raise ValueError(f"y_timestamp length at index {i} should equal pred_len={pred_len}, got {y_stamp.shape[0]}.") x_mean, x_std = np.mean(x, axis=0), np.std(x, axis=0) - x_norm = (x - x_mean) / (x_std + 1e-5) + x_std = self._floored_std(x_std) + x_norm = (x - x_mean) / x_std x_norm = np.clip(x_norm, -self.clip, self.clip) x_list.append(x_norm) @@ -639,22 +741,35 @@ def predict_batch(self, df_list, x_timestamp_list, y_timestamp_list, pred_len, T seq_lens.append(x_norm.shape[0]) y_lens.append(y_stamp.shape[0]) - # Require all series to have consistent historical and prediction lengths for batch processing - if len(set(seq_lens)) != 1: - raise ValueError(f"Parallel prediction requires all series to have consistent historical lengths, got: {seq_lens}") + # Prediction lengths must match so one autoregressive loop covers the batch. if len(set(y_lens)) != 1: raise ValueError(f"Parallel prediction requires all series to have consistent prediction lengths, got: {y_lens}") - x_batch = np.stack(x_list, axis=0).astype(np.float32) # (B, seq_len, feat) - x_stamp_batch = np.stack(x_stamp_list, axis=0).astype(np.float32) # (B, seq_len, time_feat) - y_stamp_batch = np.stack(y_stamp_list, axis=0).astype(np.float32) # (B, pred_len, time_feat) + # Left-pad heterogeneous histories to the longest series and build a + # padding mask (1 = padding) so attention never touches padded columns. + max_len = max(seq_lens) + feat_dim = x_list[0].shape[1] + time_feat_dim = x_stamp_list[0].shape[1] + x_batch = np.zeros((num_series, max_len, feat_dim), dtype=np.float32) + x_stamp_batch = np.zeros((num_series, max_len, time_feat_dim), dtype=np.float32) + pad_mask = np.zeros((num_series, max_len), dtype=np.bool_) + for i in range(num_series): + pad_len = max_len - seq_lens[i] + if pad_len > 0: + x_batch[i, pad_len:] = x_list[i] + x_stamp_batch[i, pad_len:] = x_stamp_list[i] + pad_mask[i, :pad_len] = True + else: + x_batch[i] = x_list[i] + x_stamp_batch[i] = x_stamp_list[i] + y_stamp_batch = np.stack(y_stamp_list, axis=0).astype(np.float32) # (B, pred_len, time_feat) - preds = self.generate(x_batch, x_stamp_batch, y_stamp_batch, pred_len, T, top_k, top_p, sample_count, verbose) - # preds: (B, pred_len, feat) + preds = self.generate(x_batch, x_stamp_batch, y_stamp_batch, pred_len, T, top_k, top_p, sample_count, verbose, padding_mask=pad_mask) + # preds has shape (B, pred_len, feat) pred_dfs = [] for i in range(num_series): - preds_i = preds[i] * (stds[i] + 1e-5) + means[i] + preds_i = preds[i] * stds[i] + means[i] pred_df = pd.DataFrame(preds_i, columns=self.price_cols + [self.vol_col, self.amt_vol], index=y_timestamp_list[i]) pred_dfs.append(pred_df) diff --git a/tests/test_kronos_audit_fixes.py b/tests/test_kronos_audit_fixes.py new file mode 100644 index 000000000..d6826a5d8 --- /dev/null +++ b/tests/test_kronos_audit_fixes.py @@ -0,0 +1,489 @@ +"""Regression tests for the Kronos code-audit remediation. + +Covers the Severity 1 and Severity 2 findings from the 2026-08-01 audit: + +- 2.1 top_k/top_p filtering chaining +- 2.2 rolling-buffer shift (no torch.roll) +- 2.4 normalization std floor +- 2.8 DDP-safe compute_loss +- 3.2/3.3 heterogeneous batch padding + attention masking +- 3.4 KRONOS_DEVICE override +- 3.5 lazy time features in QlibTestDataset (no input mutation) +- 2.10/3.7 config hygiene (COMET env fallback, renamed sample counters, market profiles) + +These tests run with tiny random-weight models and never download weights. +""" + +import importlib.util +import sys +import types +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import torch +import torch.nn.functional as functional +from model import Kronos, KronosPredictor, KronosTokenizer +from model.kronos import top_k_top_p_filtering + +DEVICE = "cpu" + +S1_BITS = 4 +S2_BITS = 4 +D_MODEL = 16 + + +# --------------------------------------------------------------------------- +# Tiny random-weight fixtures (no HuggingFace download) +# --------------------------------------------------------------------------- +def make_tiny_tokenizer(): + return KronosTokenizer( + d_in=6, + d_model=D_MODEL, + n_heads=4, + ff_dim=32, + n_enc_layers=2, + n_dec_layers=2, + ffn_dropout_p=0.0, + attn_dropout_p=0.0, + resid_dropout_p=0.0, + s1_bits=S1_BITS, + s2_bits=S2_BITS, + beta=0.25, + gamma0=0.01, + gamma=0.01, + zeta=0.01, + group_size=8, + ) + + +def make_tiny_predictor_model(): + return Kronos( + s1_bits=S1_BITS, + s2_bits=S2_BITS, + n_layers=1, + d_model=D_MODEL, + n_heads=4, + ff_dim=32, + ffn_dropout_p=0.0, + attn_dropout_p=0.0, + resid_dropout_p=0.0, + token_dropout_p=0.0, + learn_te=False, + ) + + +@pytest.fixture(scope="module") +def tiny_models(): + torch.manual_seed(0) + tokenizer = make_tiny_tokenizer().eval() + model = make_tiny_predictor_model().eval() + return tokenizer, model + + +def make_price_df(n, seed=0): + rng = np.random.default_rng(seed) + close = 100.0 + np.cumsum(rng.normal(0, 1.0, n)) + open_ = close - rng.normal(0, 0.1, n) + high = np.maximum(open_, close) + np.abs(rng.normal(0, 0.2, n)) + low = np.minimum(open_, close) - np.abs(rng.normal(0, 0.2, n)) + return pd.DataFrame({ + "open": open_, + "high": high, + "low": low, + "close": close, + "volume": rng.integers(100, 1000, n).astype(float), + "amount": rng.uniform(1e4, 1e5, n), + }) + + +def make_timestamps(n, start="2024-01-01"): + return pd.Series(pd.date_range(start, periods=n, freq="5min")) + + +# --------------------------------------------------------------------------- +# 2.1 top_k/top_p chaining +# --------------------------------------------------------------------------- +def _reference_filter(logits, top_k, top_p, min_tokens_to_keep=1): + out = logits.clone() + if top_k > 0: + k = min(max(top_k, min_tokens_to_keep), out.size(-1)) + keep = torch.topk(out, k)[0][..., -1, None] + out = out.masked_fill(out < keep, -float("Inf")) + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(out, descending=True) + cum = torch.cumsum(functional.softmax(sorted_logits, dim=-1), dim=-1) + remove = cum > top_p + remove[..., 1:] = remove[..., :-1].clone() + remove[..., 0] = 0 + inds = remove.scatter(1, sorted_indices, remove) + out = out.masked_fill(inds, -float("Inf")) + return out + + +def test_top_k_top_p_filtering_chains_both(): + torch.manual_seed(7) + logits = torch.randn(4, 64) + got = top_k_top_p_filtering(logits.clone(), top_k=10, top_p=0.6) + expected = _reference_filter(logits, top_k=10, top_p=0.6) + torch.testing.assert_close(got, expected) + + +def test_top_k_top_p_filtering_matches_top_k_only_when_top_p_default(): + torch.manual_seed(8) + logits = torch.randn(3, 32) + got = top_k_top_p_filtering(logits.clone(), top_k=3, top_p=1.0) + expected = _reference_filter(logits, top_k=3, top_p=1.0) + torch.testing.assert_close(got, expected) + + +def test_top_k_top_p_filtering_matches_top_p_only_when_top_k_default(): + torch.manual_seed(9) + logits = torch.randn(3, 32) + got = top_k_top_p_filtering(logits.clone(), top_k=0, top_p=0.7) + expected = _reference_filter(logits, top_k=0, top_p=0.7) + torch.testing.assert_close(got, expected) + + +def test_top_k_top_p_filtering_with_both_removes_more_than_top_k_alone(): + torch.manual_seed(10) + logits = torch.randn(1, 16) + both = top_k_top_p_filtering(logits.clone(), top_k=5, top_p=0.4) + k_only = top_k_top_p_filtering(logits.clone(), top_k=5, top_p=1.0) + kept_both = (both != -float("Inf")).sum().item() + kept_k = (k_only != -float("Inf")).sum().item() + assert 0 < kept_both < kept_k + + +# --------------------------------------------------------------------------- +# 2.4 normalization std floor +# --------------------------------------------------------------------------- +def test_floored_std_floor_applies_to_flat_columns(): + std = np.array([0.0, 1e-9, 1e-8, 0.5, 2.0]) + floored = KronosPredictor._floored_std(std) + # Flat columns are raised to the floor; normal columns keep std + eps. + np.testing.assert_array_equal(floored[:3], np.full(3, 1e-3)) + np.testing.assert_allclose(floored[3:], np.array([0.50001, 2.00001])) + + +def test_floored_std_leaves_normal_columns_untouched(): + rng = np.random.default_rng(1) + std = rng.uniform(0.01, 5.0, 8) + np.testing.assert_allclose(KronosPredictor._floored_std(std), std, atol=1e-4) + + +def test_predict_on_flat_window_is_finite(tiny_models): + tokenizer, model = tiny_models + n = 32 + flat = pd.DataFrame({ + "open": np.full(n, 100.0), + "high": np.full(n, 100.0), + "low": np.full(n, 100.0), + "close": np.full(n, 100.0), + "volume": np.full(n, 1000.0), + "amount": np.full(n, 100000.0), + }) + df = flat + ts = make_timestamps(n) + predictor = KronosPredictor(model, tokenizer, device=DEVICE, max_context=16) + with torch.no_grad(): + pred = predictor.predict( + df=df, x_timestamp=ts, y_timestamp=ts[:4], pred_len=4, + T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=False, + ) + assert np.isfinite(pred.to_numpy()).all() + + +# --------------------------------------------------------------------------- +# 2.8 compute_loss exposure +# --------------------------------------------------------------------------- +def test_compute_loss_matches_head(tiny_models): + _, model = tiny_models + s1_targets = torch.randint(0, 2 ** S1_BITS, (2, 16)) + s2_targets = torch.randint(0, 2 ** S2_BITS, (2, 16)) + s1_logits = torch.randn(2, 16, 2 ** S1_BITS) + s2_logits = torch.randn(2, 16, 2 ** S2_BITS) + expected = model.head.compute_loss(s1_logits, s2_logits, s1_targets, s2_targets) + got = model.compute_loss(s1_logits, s2_logits, s1_targets, s2_targets) + assert len(got) == 3 + for a, b in zip(got, expected, strict=True): + torch.testing.assert_close(a, b) + + +def test_compute_loss_is_finite(tiny_models): + _, model = tiny_models + s1_targets = torch.randint(0, 2 ** S1_BITS, (2, 8)) + s2_targets = torch.randint(0, 2 ** S2_BITS, (2, 8)) + s1_logits = torch.randn(2, 8, 2 ** S1_BITS) + s2_logits = torch.randn(2, 8, 2 ** S2_BITS) + loss, s1, s2 = model.compute_loss(s1_logits, s2_logits, s1_targets, s2_targets) + assert torch.isfinite(loss) and torch.isfinite(s1) and torch.isfinite(s2) + + +# --------------------------------------------------------------------------- +# 3.4 device selection +# --------------------------------------------------------------------------- +def test_device_env_override(monkeypatch, tiny_models): + tokenizer, model = tiny_models + monkeypatch.setenv("KRONOS_DEVICE", "cpu") + predictor = KronosPredictor(model, tokenizer, device=None, max_context=16) + assert predictor.device == "cpu" + + +def test_device_accepts_torch_device(tiny_models): + tokenizer, model = tiny_models + predictor = KronosPredictor(model, tokenizer, device=torch.device("cpu"), max_context=16) + assert predictor.device == "cpu" + + +def test_device_explicit_wins_over_env(monkeypatch, tiny_models): + tokenizer, model = tiny_models + monkeypatch.setenv("KRONOS_DEVICE", "cuda:0") + predictor = KronosPredictor(model, tokenizer, device="cpu", max_context=16) + assert predictor.device == "cpu" + + +# --------------------------------------------------------------------------- +# 2.2/3.2/3.3 end-to-end inference with tiny models +# --------------------------------------------------------------------------- +def test_autoregressive_inference_smoke(tiny_models): + tokenizer, model = tiny_models + predictor = KronosPredictor(model, tokenizer, device=DEVICE, max_context=16) + df = make_price_df(64) + ts = make_timestamps(64) + with torch.no_grad(): + pred = predictor.predict( + df=df, x_timestamp=ts, y_timestamp=ts[:4], pred_len=4, + T=1.0, top_k=0, top_p=0.9, sample_count=2, verbose=False, + ) + assert pred.shape == (4, 6) + assert np.isfinite(pred.to_numpy()).all() + + +def test_autoregressive_inference_exercises_shift_branch(tiny_models): + """seq_len > max_context forces the rolling-buffer shift path (2.2).""" + tokenizer, model = tiny_models + predictor = KronosPredictor(model, tokenizer, device=DEVICE, max_context=8) + df = make_price_df(64) + ts = make_timestamps(64) + with torch.no_grad(): + pred = predictor.predict( + df=df, x_timestamp=ts, y_timestamp=ts[:3], pred_len=3, + T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=False, + ) + assert pred.shape == (3, 6) + assert np.isfinite(pred.to_numpy()).all() + + +def test_predict_batch_heterogeneous_lengths(tiny_models): + tokenizer, model = tiny_models + predictor = KronosPredictor(model, tokenizer, device=DEVICE, max_context=16) + + df_long = make_price_df(64) + df_short = make_price_df(40, seed=1) + ts_long = make_timestamps(64) + ts_short = make_timestamps(40, start="2024-02-01") + + with torch.no_grad(): + preds = predictor.predict_batch( + df_list=[df_long, df_short], + x_timestamp_list=[ts_long, ts_short], + y_timestamp_list=[ts_long[:4], ts_short[:4]], + pred_len=4, + T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=False, + ) + assert len(preds) == 2 + assert preds[0].shape == (4, 6) + assert preds[1].shape == (4, 6) + assert np.isfinite(preds[0].to_numpy()).all() + assert np.isfinite(preds[1].to_numpy()).all() + + +def test_predict_batch_single_length_matches_old_contract(tiny_models): + """Equal-length batches must still work without padding (backward compat).""" + tokenizer, model = tiny_models + predictor = KronosPredictor(model, tokenizer, device=DEVICE, max_context=16) + df_a = make_price_df(48) + df_b = make_price_df(48, seed=2) + ts_a = make_timestamps(48) + ts_b = make_timestamps(48, start="2024-03-01") + with torch.no_grad(): + preds = predictor.predict_batch( + df_list=[df_a, df_b], + x_timestamp_list=[ts_a, ts_b], + y_timestamp_list=[ts_a[:4], ts_b[:4]], + pred_len=4, + T=1.0, top_k=0, top_p=0.9, sample_count=1, verbose=False, + ) + assert len(preds) == 2 + assert all(p.shape == (4, 6) for p in preds) + + +def test_predict_batch_rejects_mismatched_pred_len(tiny_models): + tokenizer, model = tiny_models + predictor = KronosPredictor(model, tokenizer, device=DEVICE, max_context=16) + df_a = make_price_df(48) + df_b = make_price_df(48, seed=2) + ts_a = make_timestamps(48) + ts_b = make_timestamps(48, start="2024-03-01") + with pytest.raises(ValueError, match="pred_len=4"): + predictor.predict_batch( + df_list=[df_a, df_b], + x_timestamp_list=[ts_a, ts_b], + y_timestamp_list=[ts_a[:4], ts_b[:6]], + pred_len=4, + ) + + +# --------------------------------------------------------------------------- +# 3.5 QlibTestDataset lazy time features (stubbed qlib/matplotlib imports) +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def qlib_test_module(): + stubs = {} + for mod_name in [ + "qlib", "qlib.config", "qlib.backtest", "qlib.backtest.backtest", + "qlib.backtest.executor", "qlib.contrib", "qlib.contrib.evaluate", + "qlib.contrib.strategy", "qlib.utils", "qlib.utils.time", + "matplotlib", "matplotlib.pyplot", + ]: + stubs[mod_name] = types.ModuleType(mod_name) + sys.modules[mod_name] = stubs[mod_name] + stubs["qlib.config"].REG_CN = "cn" + stubs["qlib.config"].REG_US = "us" + for m in stubs.values(): + m.__getattr__ = lambda name: types.SimpleNamespace() + + root = str(Path(__file__).resolve().parents[1]) + finetune = str(Path(__file__).resolve().parents[1] / "finetune") + for p in (root, finetune): + if p not in sys.path: + sys.path.insert(0, p) + + spec = importlib.util.spec_from_file_location( + "qlib_test_stub", str(Path(finetune) / "qlib_test.py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + yield module + + # Restore sys.modules so real qlib/matplotlib imports are unaffected later. + for name in stubs: + sys.modules.pop(name, None) + for p in (root, finetune): + if p in sys.path: + sys.path.remove(p) + + +def _sample_config(**overrides): + cfg = types.SimpleNamespace() + cfg.lookback_window = 10 + cfg.predict_window = 3 + cfg.clip = 5.0 + cfg.feature_list = ["open", "high", "low", "close", "vol", "amt"] + cfg.time_feature_list = ["minute", "hour", "weekday", "day", "month"] + for k, v in overrides.items(): + setattr(cfg, k, v) + return cfg + + +def _synthetic_series_data(n=50): + idx = pd.DatetimeIndex(pd.date_range("2024-01-01", periods=n, freq="D"), name="datetime") + df = pd.DataFrame({ + "open": np.linspace(1, 2, n), + "high": np.linspace(1.1, 2.1, n), + "low": np.linspace(0.9, 1.9, n), + "close": np.linspace(1.05, 2.05, n), + "vol": np.full(n, 100.0), + "amt": np.full(n, 1000.0), + }, index=idx) + return df + + +def test_qlib_dataset_does_not_mutate_input(qlib_test_module): + dataset_cls = qlib_test_module.QlibTestDataset + data = {"A": _synthetic_series_data(), "B": _synthetic_series_data(60)} + original_cols = {k: list(v.columns) for k, v in data.items()} + + ds = dataset_cls(data=data, config=_sample_config()) + # Force feature computation for both symbols. + for i in range(len(ds)): + ds[i] + + for k, cols in original_cols.items(): + assert list(data[k].columns) == cols + assert "minute" not in data[k].columns + + +def test_qlib_dataset_getitem_shapes(qlib_test_module): + dataset_cls = qlib_test_module.QlibTestDataset + cfg = _sample_config() + data = {"A": _synthetic_series_data()} + ds = dataset_cls(data=data, config=cfg) + x, x_stamp, y_stamp, symbol, timestamp = ds[0] + assert x.shape == (cfg.lookback_window, len(cfg.feature_list)) + assert x_stamp.shape == (cfg.lookback_window, len(cfg.time_feature_list)) + assert y_stamp.shape == (cfg.predict_window, len(cfg.time_feature_list)) + assert symbol == "A" + assert np.isfinite(x.numpy()).all() + + +def test_qlib_dataset_flat_window_normalization(qlib_test_module): + dataset_cls = qlib_test_module.QlibTestDataset + data = {"A": _synthetic_series_data()} + ds = dataset_cls(data=data, config=_sample_config()) + x, _, _, _, _ = ds[len(data["A"]) - ds.window_size] # last valid window + assert np.isfinite(x.numpy()).all() + + +# --------------------------------------------------------------------------- +# Config hygiene (2.10, 3.7, 2.5) - loaded standalone +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def config_module(): + path = str(Path(__file__).resolve().parents[1] / "finetune" / "config.py") + spec = importlib.util.spec_from_file_location("kronos_config_standalone", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_config_uses_comet_env_fallback(monkeypatch, config_module): + monkeypatch.delenv("COMET_API_KEY", raising=False) + monkeypatch.delenv("COMET_WORKSPACE", raising=False) + cfg = config_module.Config() + assert cfg.comet_config["api_key"] == "" + + +def test_config_reads_comet_env(monkeypatch, config_module): + monkeypatch.setenv("COMET_API_KEY", "secret-key") + cfg = config_module.Config() + assert cfg.comet_config["api_key"] == "secret-key" + + +def test_config_sample_counters_renamed(config_module): + cfg = config_module.Config() + assert hasattr(cfg, "n_train_samples_per_epoch") + assert hasattr(cfg, "n_val_samples_per_epoch") + assert not hasattr(cfg, "n_train_iter") + + +def test_config_market_profiles(config_module): + cfg = config_module.Config() + cn = cfg._set_exchange_kwargs("cn", "day") + assert cn["limit_threshold"] == 0.095 + assert cn["deal_price"] == "open" + generic = cfg._set_exchange_kwargs("generic", "1min") + assert generic["freq"] == "1min" + assert generic["limit_threshold"] is None + assert generic["deal_price"] == "close" + with pytest.raises(ValueError): + cfg._set_exchange_kwargs("nope", "day") + + +def test_config_default_backtest_params(config_module): + cfg = config_module.Config() + assert cfg.backtest_exchange_kwargs["freq"] == cfg.backtest_freq + assert cfg.inference_num_workers <= 8 + assert cfg.qlib_region == "cn"