LLM Optimizer Tuner

What It Does

Installation

pip install pyyaml numpy pandas matplotlib

Usage

python llm_optimizer_tuner.py <config_file> <tasks_file> [--output <output_file>]

python llm_optimizer_tuner.py config.yaml tasks.json --output metrics_report.csv

Source Code

import argparse
import yaml
import json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

def simulate_workload(config, tasks):
    """
    Simulates workloads based on routing configurations and tasks.

    Args:
        config (dict): Routing configuration.
        tasks (list): List of tasks to simulate.

    Returns:
        pd.DataFrame: DataFrame containing performance metrics.
    """
    metrics = []

    for task in tasks:
        task_type = task.get('type', 'default')
        model = config.get(task_type, {}).get('model', 'default')
        latency = np.random.uniform(0.5, 2.0)  # Simulated latency
        cost = np.random.uniform(0.01, 0.1)    # Simulated cost
        metrics.append({
            'task': task.get('name', 'unknown'),
            'model': model,
            'latency': latency,
            'cost': cost
        })

    return pd.DataFrame(metrics)

def generate_report(metrics, output_file):
    """
    Generates a CSV report of performance metrics.

    Args:
        metrics (pd.DataFrame): DataFrame containing performance metrics.
        output_file (str): Path to save the report.
    """
    metrics.to_csv(output_file, index=False, encoding='utf-8')
    print(f"Report saved to {output_file}")

def optimize(config_file, tasks_file, output_file='report.csv'):
    """
    Main function to optimize routing configurations.

    Args:
        config_file (str): Path to the routing configuration file.
        tasks_file (str): Path to the sample tasks file.
        output_file (str): Path to save the performance metrics report.
    """
    try:
        with open(config_file, 'r') as f:
            config = yaml.safe_load(f)

        with open(tasks_file, 'r') as f:
            tasks = json.load(f)

        if not isinstance(tasks, list):
            raise ValueError("Tasks file must contain a list of tasks.")

        metrics = simulate_workload(config, tasks)
        generate_report(metrics, output_file)

        print("Optimization complete. Metrics:")
        print(metrics)

    except FileNotFoundError as e:
        print(f"Error: {e}")
    except yaml.YAMLError as e:
        print(f"Error parsing YAML file: {e}")
    except json.JSONDecodeError as e:
        print(f"Error parsing JSON file: {e}")
    except ValueError as e:
        print(f"Error: {e}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="LLM Optimizer Tuner")
    parser.add_argument("config", help="Path to the routing configuration file (YAML)")
    parser.add_argument("tasks", help="Path to the sample tasks file (JSON)")
    parser.add_argument("--output", default="report.csv", help="Path to save the performance metrics report")

    args = parser.parse_args()

    optimize(args.config, args.tasks, args.output)