Token Efficiency Optimizer

What It Does

Installation

pip install numpy transformers

Usage

python token_efficiency_optimizer.py "This is a test input." --max_memory 4 --tokenizer bert-base-uncased

Source Code

import numpy as np
from transformers import PreTrainedTokenizerBase

def optimize_tokens(input_text, tokenizer: PreTrainedTokenizerBase, max_memory: int = 8):
    """
    Optimize token handling by splitting input text into manageable token batches
    based on memory constraints.

    Args:
        input_text (str): The raw text input to be tokenized and optimized.
        tokenizer (PreTrainedTokenizerBase): A Hugging Face tokenizer instance.
        max_memory (int): Maximum memory available in GB. Default is 8 GB.

    Returns:
        list: A list of token batches optimized for processing.
    """
    if not input_text:
        raise ValueError("Input text cannot be empty.")

    if max_memory <= 0:
        raise ValueError("Max memory must be greater than 0.")

    # Estimate the token limit based on memory constraints
    tokens_per_gb = 100000  # Assumption: 100,000 tokens per GB
    max_tokens = max_memory * tokens_per_gb

    # Tokenize the input text
    tokenized = tokenizer(input_text, return_tensors="np", truncation=False, add_special_tokens=False)
    input_ids = tokenized["input_ids"]

    if isinstance(input_ids, np.ndarray):
        input_ids = input_ids.squeeze()
    else:
        input_ids = np.array(input_ids[0])  # Ensure input_ids is a flat array

    # Split tokens into manageable batches
    token_batches = np.array_split(input_ids, max(1, np.ceil(len(input_ids) / max_tokens)))

    # Convert batches back to lists
    optimized_batches = [batch.tolist() for batch in token_batches]

    return optimized_batches

if __name__ == "__main__":
    import argparse
    from transformers import AutoTokenizer

    parser = argparse.ArgumentParser(description="Token Efficiency Optimizer")
    parser.add_argument("input_text", type=str, help="Raw text input to optimize.")
    parser.add_argument("--max_memory", type=int, default=8, help="Maximum memory in GB (default: 8).")
    parser.add_argument("--tokenizer", type=str, required=True, help="Pretrained tokenizer model name (e.g., 'bert-base-uncased').")

    args = parser.parse_args()

    # Load the tokenizer
    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)

    # Optimize tokens
    try:
        optimized_batches = optimize_tokens(args.input_text, tokenizer, args.max_memory)
        print("Optimized token batches:")
        for i, batch in enumerate(optimized_batches):
            print(f"Batch {i + 1}: {batch}")
    except Exception as e:
        print(f"Error: {e}")