Benchmark: Model benchmark - deterministic training support

docs/user-tutorial/benchmarks/model-benchmarks.md

@@ -34,6 +34,18 @@ For inference, supported percentiles include
 
 **New: Support fp8_hybrid and fp8_e4m3 precision for BERT models.**
 
+**New: SDC Support**
+SuperBench now supports SDC to ensure reproducibility across runs. This includes fixed seeds and deterministic algorithms. To enable SDC, the following flags and environment variables must be set:
+
+- **Flags:**
+  - `--deterministic`: Enables deterministic computation.
+  - `--deterministic_seed <seed>`: Sets the seed for reproducibility.
+  - `--generate_log` : Generates the log file that can be used as reference for comparison
+  - `--compare_log <path>`: Specifies the path to the reference log for comparison.
+
+- **Environment Variables:**
+  - `CUBLAS_WORKSPACE_CONFIG=:4096:8`: Ensures deterministic behavior in cuBLAS.
+
 #### Metrics
 
 | Name                                                                                    | Unit                   | Description                                                                  |

examples/benchmarks/pytorch_deterministic_example.py

@@ -0,0 +1,102 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Unified PyTorch deterministic training example for all supported models.
+
+Commands to run:
+Generate log:
+
+CUBLAS_WORKSPACE_CONFIG=:4096:8 python3 examples/benchmarks/pytorch_deterministic_example.py
+--model <model_from_MODEL_CHOICES> --generate-log ./outputs/determinism_ref.json
+
+Compare log:
+
+CUBLAS_WORKSPACE_CONFIG=:4096:8 python3 examples/benchmarks/pytorch_deterministic_example.py
+--model <model_from_MODEL_CHOICES> --compare-log ./outputs/determinism_ref.json
+
+"""
+
+import argparse
+from superbench.benchmarks import BenchmarkRegistry, Framework
+from superbench.common.utils import logger
+
+MODEL_CHOICES = [
+    'bert-large',
+    'gpt2-small',
+    'llama2-7b',
+    'mixtral-8x7b',
+    'resnet101',
+    'lstm',
+]
+
+DEFAULT_PARAMS = {
+    'bert-large':
+    '--batch_size 1 --seq_len 64 --num_warmup 1 --num_steps 200 --precision float32 '
+    '--model_action train --deterministic --deterministic_seed 42 --check_frequency 20',
+    'gpt2-small':
+    '--batch_size 1 --num_steps 300 --num_warmup 1 --seq_len 128 --precision float32 '
+    '--model_action train --deterministic --deterministic_seed 42 --check_frequency 20',
+    'llama2-7b':
+    '--batch_size 1 --num_steps 300 --num_warmup 1 --seq_len 512 --precision float32 --model_action train '
+    '--deterministic --deterministic_seed 42 --check_frequency 20',
+    'mixtral-8x7b':
+    '--hidden_size=4096 --num_hidden_layers=32 --num_attention_heads=32 --intermediate_size=14336 '
+    '--num_key_value_heads=8 --max_position_embeddings=32768 --router_aux_loss_coef=0.02 '
+    '--deterministic --deterministic_seed 42 --check_frequency 20',
+    'resnet101':
+    '--batch_size 1 --precision float32 --num_warmup 1 --num_steps 120 --sample_count 8192 '
+    '--pin_memory --model_action train --deterministic --deterministic_seed 42 --check_frequency 20',
+    'lstm':
+    '--batch_size 1 --num_steps 100 --num_warmup 1 --seq_len 64 --precision float16 '
+    '--model_action train --deterministic --deterministic_seed 42 --check_frequency 20',
+}
+
+
+def main():
+    """Main function for determinism example file."""
+    parser = argparse.ArgumentParser(description='Unified PyTorch deterministic training example.')
+    parser.add_argument('--model', type=str, choices=MODEL_CHOICES, required=True, help='Model to run.')
+    parser.add_argument(
+        '--generate-log',
+        nargs='?',
+        const=True,
+        default=None,
+        help='Enable fingerprint log generation. Optionally specify a path to save the log.',
+    )
+    parser.add_argument(
+        '--compare-log',
+        type=str,
+        default=None,
+        help='Path to reference fingerprint log for comparison.',
+    )
+    parser.add_argument(
+        '--deterministic-seed',
+        type=int,
+        default=42,
+        help='Seed for deterministic training.',
+    )
+    args = parser.parse_args()
+
+    parameters = DEFAULT_PARAMS[args.model]
+    if args.deterministic_seed:
+        parameters += f' --deterministic_seed {args.deterministic_seed}'
+    if args.generate_log:
+        parameters += ' --generate-log'
+        if isinstance(args.generate_log, str):
+            parameters += f' {args.generate_log}'
+    if args.compare_log:
+        parameters += f' --compare-log {args.compare_log}'
+
+    context = BenchmarkRegistry.create_benchmark_context(args.model, parameters=parameters, framework=Framework.PYTORCH)
+    benchmark = BenchmarkRegistry.launch_benchmark(context)
+    logger.info(f'Benchmark finished. Return code: {benchmark.return_code}')
+    if hasattr(benchmark, '_model_run_metadata'):
+        logger.info(f'Run metadata: {benchmark._model_run_metadata}')
+    if hasattr(benchmark, '_model_run_losses'):
+        logger.info(f'Losses: {benchmark._model_run_losses[:5]} ...')
+    if hasattr(benchmark, '_model_run_periodic'):
+        logger.info(f'Periodic: {benchmark._model_run_periodic}')
+
+
+if __name__ == '__main__':
+    main()

superbench/benchmarks/base.py

@@ -110,14 +110,95 @@ def parse_args(self, ignore_invalid=False):
                 logger.error('Invalid argument - benchmark: {}, message: {}.'.format(self._name, str(e)))
                 return False, None, []
 
+        if args is not None and 'compare_log' in [a.dest for a in self._parser._actions]:
+            args = self._parse_args_override_step(args)
+
         ret = True
+        ret = self._check_unknown_args(unknown)
+
+        return ret, args, unknown
+
+    def _parse_args_override_step(self, args):
+        """Override arguments using metadata from a compare log file.
+
+        Args:
+            args: Parsed arguments.
+
+        Returns:
+            argparse.Namespace: Updated arguments with overridden values.
+        """
+        return self._override_args_with_compare_log(args)
+
+    def _override_args_with_compare_log(self, args):
+        """Override arguments with metadata from a compare log file if available.
+
+        Args:
+            args: Parsed arguments.
+
+        Returns:
+            argparse: Arguments updated with metadata values.
+        """
+        # Only override if compare_log is set and is a valid argument for this benchmark
+        if args is not None and hasattr(args, 'compare_log') and getattr(args, 'compare_log', None):
+            logger.info(f'Original Arguments before overriding from compare_log metadata for determinism: {args}')
+            try:
+                from superbench.common import model_log_utils
+                log_data = model_log_utils.load_model_log(args.compare_log)
+                metadata = log_data.get('metadata', {})
+                try:
+                    from superbench.benchmarks import Precision
+                except ImportError:
+                    Precision = None
+                for key, value in metadata.items():
+                    if hasattr(args, key):
+                        if key == 'precision' and Precision is not None:
+                            setattr(args, key, self._convert_precision_value(value, Precision))
+                        else:
+                            setattr(args, key, value)
+                logger.info(f'Arguments overridden from compare_log metadata for determinism. New Arguments: {args}')
+            except Exception as e:
+                logger.info(f'Failed to override args from compare_log metadata: {e}')
+        return args
+
+    def _convert_precision_value(self, value, Precision):
+        """Convert precision values to the appropriate format.
+
+        Args:
+            value: The precision value to convert.
+            Precision: The Precision class or type to convert to.
+
+        Returns:
+            list: A list of converted precision values.
+        """
+        if isinstance(value, list):
+            converted = []
+            for v in value:
+                if isinstance(v, Precision):
+                    converted.append(v)
+                else:
+                    converted.append(Precision(v))
+            return converted
+        else:
+            if isinstance(value, Precision):
+                return [value]
+            else:
+                return [Precision(value)]
+
+    def _check_unknown_args(self, unknown):
+        """Check for unknown arguments and log an error if any are found.
+
+        Args:
+            unknown (list): List of unknown arguments.
+
+        Returns:
+            bool: False if unknown arguments are found, True otherwise.
+        """
         if len(unknown) > 0:
             logger.error(
                 'Unknown arguments - benchmark: {}, unknown arguments: {}'.format(self._name, ' '.join(unknown))
             )
-            ret = False
-
-        return ret, args, unknown
+            return False
+        return True
 
     def _preprocess(self):
         """Preprocess/preparation operations before the benchmarking.

superbench/benchmarks/model_benchmarks/model_base.py

@@ -186,6 +186,17 @@ def _generate_dataset(self):
         """
         pass
 
+    def set_deterministic_seed(self):
+        """Hook to set deterministic RNG state before dataset generation.
+
+        Default implementation is a no-op. Framework-specific subclasses may
+        override this to apply deterministic RNG settings (for example,
+        PyTorch benchmarks implement this to call their deterministic setup
+        when requested). This is called from _preprocess() before
+        _generate_dataset().
+        """
+        return None
+
     @abstractmethod
     def _init_dataloader(self):
         """Initialize the dataloader.
@@ -221,6 +232,12 @@ def _preprocess(self):
             self._result.set_return_code(ReturnCode.DISTRIBUTED_SETTING_INIT_FAILURE)
             return False
 
+        # Invoke model-specific deterministic seeding hook before dataset generation
+        try:
+            self.set_deterministic_seed()
+        except Exception:
+            logger.info('set_deterministic_seed() hook failed or not implemented for model: %s', self._name)
+
         # Set sample_count aligned with batch_size.
         self._args.sample_count = math.ceil(self._args.sample_count / self._args.batch_size) * self._args.batch_size