Adaptive LLM Optimizer

What It Does

Installation

git clone https://github.com/your-repo/adaptive-llm-optimizer.git
   cd adaptive-llm-optimizer

pip install -r requirements.txt

Usage

python adaptive_llm_optimizer.py --model_path model.pt --hardware cpu --initial_batch_size 32 --precision FP32

python adaptive_llm_optimizer.py --model_path model.pt --hardware gpu --initial_batch_size 64 --precision FP16

Source Code

import argparse
import torch
import psutil
import time


def profile_hardware():
    """Profiles the hardware capabilities."""
    cpu_count = psutil.cpu_count(logical=True)
    memory = psutil.virtual_memory().total / (1024 ** 3)  # Convert to GB
    gpu_available = torch.cuda.is_available()
    gpu_name = torch.cuda.get_device_name(0) if gpu_available else "None"
    return {
        "cpu_count": cpu_count,
        "memory_gb": memory,
        "gpu_available": gpu_available,
        "gpu_name": gpu_name,
    }


def adjust_settings(hardware_profile, initial_batch_size, precision):
    """Adjusts inference settings based on hardware profile."""
    settings = {
        "batch_size": initial_batch_size,
        "precision": precision,
        "threads": hardware_profile["cpu_count"] // 2,
    }

    if hardware_profile["gpu_available"]:
        settings["precision"] = "FP16" if precision == "FP32" else precision
    else:
        settings["precision"] = "FP32"

    return settings


def monitor_performance(model, settings):
    """Simulates real-time monitoring of inference performance."""
    # Simulate latency and throughput metrics
    latency = 1.0 / settings["batch_size"]  # Simplified example
    throughput = settings["batch_size"] / latency
    return {
        "latency": latency,
        "throughput": throughput,
    }


def optimize_model(model_path, hardware, initial_batch_size, precision):
    """Main function to optimize the model settings."""
    hardware_profile = profile_hardware()
    print(f"Hardware Profile: {hardware_profile}")

    settings = adjust_settings(hardware_profile, initial_batch_size, precision)
    print(f"Initial Settings: {settings}")

    # Simulate loading the model (mocked for simplicity)
    model = "MockModel"  # Replace with actual model loading in real use

    performance = monitor_performance(model, settings)
    print(f"Performance Metrics: {performance}")

    return settings, performance


def main():
    parser = argparse.ArgumentParser(description="Adaptive LLM Optimizer")
    parser.add_argument("--model_path", type=str, required=True, help="Path to the model file")
    parser.add_argument("--hardware", type=str, choices=["cpu", "gpu"], required=True, help="Hardware type")
    parser.add_argument("--initial_batch_size", type=int, default=32, help="Initial batch size")
    parser.add_argument("--precision", type=str, choices=["FP16", "FP32"], default="FP32", help="Precision")

    args = parser.parse_args()

    settings, performance = optimize_model(
        args.model_path, args.hardware, args.initial_batch_size, args.precision
    )

    print("Optimized Settings:", settings)
    print("Performance Summary:", performance)


if __name__ == "__main__":
    main()