Quantization Simulator

What It Does

Installation

pip install torch pytest

Usage

python quantization_simulator.py <model_path> --quantization_levels 16 8 4

python quantization_simulator.py my_model.pth --quantization_levels 16 8

import torch
from quantization_simulator import simulate_quantization

model = torch.nn.Linear(10, 2)
results = simulate_quantization(model, quantization_levels=[16, 8])
print(results)

Source Code

import torch
import numpy as np
from typing import List, Optional, Tuple, Union

def simulate_quantization(
    model: torch.nn.Module,
    eval_data: Optional[torch.utils.data.DataLoader] = None,
    quantization_levels: List[int] = [16, 8, 4],
) -> dict:
    """
    Simulates the effects of quantization on a PyTorch model.

    Args:
        model (torch.nn.Module): The PyTorch model to be quantized.
        eval_data (Optional[torch.utils.data.DataLoader]): Evaluation dataset to test accuracy degradation.
        quantization_levels (List[int]): List of quantization bit levels to simulate.

    Returns:
        dict: A dictionary containing memory usage, inference speed, and accuracy degradation metrics.
    """
    if not isinstance(model, torch.nn.Module):
        raise ValueError("The 'model' argument must be a PyTorch model.")

    results = {}

    for bits in quantization_levels:
        if bits not in [16, 8, 4]:
            raise ValueError(f"Unsupported quantization level: {bits}. Supported levels are 16, 8, and 4.")

        # Simulate quantization
        scale = 2 ** bits - 1
        quantized_model = torch.quantization.quantize_dynamic(
            model, {torch.nn.Linear}, dtype=torch.qint8 if bits == 8 else torch.float16
        )

        # Measure memory usage
        memory_usage = sum(p.numel() * p.element_size() for p in quantized_model.parameters()) / 1e6

        # Measure inference speed
        dummy_input = torch.randn(1, *list(model.parameters())[0].shape[1:])
        inference_time = None
        try:
            if torch.cuda.is_available():
                start_time = torch.cuda.Event(enable_timing=True)
                end_time = torch.cuda.Event(enable_timing=True)
                start_time.record()
                _ = quantized_model(dummy_input)
                end_time.record()
                torch.cuda.synchronize()
                inference_time = start_time.elapsed_time(end_time)
        except RuntimeError:
            pass

        # Measure accuracy degradation (if eval_data is provided)
        accuracy = None
        if eval_data:
            correct = 0
            total = 0
            with torch.no_grad():
                for inputs, labels in eval_data:
                    outputs = quantized_model(inputs)
                    _, predicted = torch.max(outputs.data, 1)
                    total += labels.size(0)
                    correct += (predicted == labels).sum().item()
            accuracy = correct / total

        # Store results
        results[bits] = {
            "memory_usage_mb": memory_usage,
            "inference_time_ms": inference_time,
            "accuracy": accuracy,
        }

    return results

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Quantization Simulator")
    parser.add_argument("model_path", type=str, help="Path to the PyTorch model file.")
    parser.add_argument(
        "--quantization_levels",
        type=int,
        nargs="+",
        default=[16, 8, 4],
        help="List of quantization levels to simulate (default: [16, 8, 4]).",
    )
    args = parser.parse_args()

    # Load model
    try:
        model = torch.load(args.model_path)
    except Exception as e:
        print(f"Error loading model: {e}")
        exit(1)

    # Run simulation
    results = simulate_quantization(model, quantization_levels=args.quantization_levels)

    # Print results
    for bits, metrics in results.items():
        print(f"Quantization Level: {bits}-bit")
        print(f"  Memory Usage (MB): {metrics['memory_usage_mb']:.2f}")
        if metrics['inference_time_ms'] is not None:
            print(f"  Inference Time (ms): {metrics['inference_time_ms']:.2f}")
        else:
            print("  Inference Time (ms): Not available (no CUDA support)")
        if metrics['accuracy'] is not None:
            print(f"  Accuracy: {metrics['accuracy']:.2%}")
        print()