LLM Efficiency Analyzer

What It Does

Installation

git clone https://github.com/your-repo/llm_efficiency_analyzer.git
   cd llm_efficiency_analyzer

pip install -r requirements.txt

Usage

Source Code

import json
import csv
import numpy as np
import matplotlib.pyplot as plt
from transformers import AutoModelForCausalLM, AutoTokenizer
import typer
from pathlib import Path
from typing import List, Tuple

app = typer.Typer(name="LLM Efficiency Analyzer", help="Evaluate compute vs. performance trade-offs for LLMs.")

def evaluate_model_performance(
    model_name: str,
    dataset_path: Path,
    seq_len_range: Tuple[int, int],
    batch_size_range: Tuple[int, int]
):
    try:
        # Load model and tokenizer
        model = AutoModelForCausalLM.from_pretrained(model_name)
        tokenizer = AutoTokenizer.from_pretrained(model_name)
    except Exception as e:
        typer.secho(f"Error loading model or tokenizer: {e}", fg=typer.colors.RED)
        raise typer.Exit(code=1)

    try:
        # Load dataset
        with open(dataset_path, "r") as f:
            data = [json.loads(line) for line in f] if dataset_path.suffix == ".jsonl" else list(csv.DictReader(f))
    except Exception as e:
        typer.secho(f"Error loading dataset: {e}", fg=typer.colors.RED)
        raise typer.Exit(code=1)

    results = []

    for seq_len in range(seq_len_range[0], seq_len_range[1] + 1, 64):
        for batch_size in range(batch_size_range[0], batch_size_range[1] + 1, 16):
            try:
                # Simulate performance evaluation
                compute_time = seq_len * batch_size * 0.01  # Placeholder for actual compute time
                performance = 1 / (1 + np.exp(-0.1 * (seq_len - 256))) * (1 / (1 + np.exp(-0.1 * (batch_size - 32))))

                results.append({
                    "seq_len": seq_len,
                    "batch_size": batch_size,
                    "compute_time": compute_time,
                    "performance": performance
                })
            except Exception as e:
                typer.secho(f"Error during evaluation: {e}", fg=typer.colors.RED)

    return results

def generate_report(results: List[dict], output_path: Path):
    try:
        with open(output_path, "w") as f:
            writer = csv.DictWriter(f, fieldnames=results[0].keys())
            writer.writeheader()
            writer.writerows(results)
    except Exception as e:
        typer.secho(f"Error writing report: {e}", fg=typer.colors.RED)
        raise typer.Exit(code=1)

def generate_plot(results: List[dict], output_path: Path):
    try:
        seq_lens = sorted(set(r["seq_len"] for r in results))
        batch_sizes = sorted(set(r["batch_size"] for r in results))

        for batch_size in batch_sizes:
            filtered = [r for r in results if r["batch_size"] == batch_size]
            x = [r["seq_len"] for r in filtered]
            y = [r["performance"] for r in filtered]
            plt.plot(x, y, label=f"Batch Size {batch_size}")

        plt.xlabel("Sequence Length")
        plt.ylabel("Performance")
        plt.title("Performance vs. Sequence Length")
        plt.legend()
        plt.savefig(output_path)
        plt.close()
    except Exception as e:
        typer.secho(f"Error generating plot: {e}", fg=typer.colors.RED)
        raise typer.Exit(code=1)

@app.command()
def analyze(
    model: str = typer.Option(..., help="Hugging Face model name or path."),
    dataset: Path = typer.Option(..., help="Path to dataset file (CSV or JSONL)."),
    seq_len_range: Tuple[int, int] = typer.Option((128, 512), help="Range of sequence lengths (min, max)."),
    batch_size_range: Tuple[int, int] = typer.Option((16, 64), help="Range of batch sizes (min, max)."),
    output_dir: Path = typer.Option("./output", help="Directory to save reports and plots.")
):
    "Analyze compute vs. performance trade-offs for a given LLM."

    output_dir.mkdir(parents=True, exist_ok=True)

    typer.secho("Starting analysis...", fg=typer.colors.GREEN)
    results = evaluate_model_performance(model, dataset, seq_len_range, batch_size_range)

    report_path = output_dir / "efficiency_report.csv"
    plot_path = output_dir / "efficiency_plot.png"

    generate_report(results, report_path)
    generate_plot(results, plot_path)

    typer.secho(f"Analysis complete. Report saved to {report_path}", fg=typer.colors.GREEN)
    typer.secho(f"Plot saved to {plot_path}", fg=typer.colors.GREEN)

if __name__ == "__main__":
    app()