Token Optimizer

What It Does

Installation

git clone <repository-url>
   cd token_optimizer

pip install -r requirements.txt

Usage

python token_optimizer.py --input_file path/to/textfile.txt

python token_optimizer.py --input_text "Your text here."

Source Code

import argparse
from nltk.tokenize import word_tokenize
from unittest.mock import MagicMock

class MockEncoding:
    """Mock encoding class to simulate tiktoken behavior."""
    def encode(self, text):
        return [ord(char) for char in text]

def analyze_token_usage(text, encoding):
    """Analyzes token usage in the given text and suggests optimizations."""
    tokens = encoding.encode(text)
    token_count = len(tokens)
    words = word_tokenize(text)

    suggestions = []
    if token_count > 100:
        suggestions.append("Consider truncating or summarizing the text to reduce token usage.")
    if len(words) > token_count:
        suggestions.append("Consider rephrasing to use fewer complex words.")

    return {
        "original_text": text,
        "token_count": token_count,
        "suggestions": suggestions
    }

def process_file(file_path, encoding):
    """Processes a file and analyzes token usage for each line."""
    try:
        with open(file_path, 'r', encoding='utf-8') as file:
            lines = file.readlines()
        results = [analyze_token_usage(line.strip(), encoding) for line in lines if line.strip()]
        return results
    except FileNotFoundError:
        print("Error: File not found.")
        return []

def main():
    """Token Optimizer: Analyze and optimize token usage in text."""
    parser = argparse.ArgumentParser(description="Token Optimizer: Analyze and optimize token usage in text.")
    parser.add_argument('--input_file', type=str, help='Path to the input text file.')
    parser.add_argument('--input_text', type=str, help='Input text string.')
    args = parser.parse_args()

    # Mock the nltk punkt tokenizer download to avoid network dependency
    from nltk.data import find
    try:
        find('tokenizers/punkt')
    except LookupError:
        from nltk import download
        download('punkt', quiet=True)

    encoding = MockEncoding()

    if args.input_file:
        results = process_file(args.input_file, encoding)
    elif args.input_text:
        results = [analyze_token_usage(args.input_text, encoding)]
    else:
        print("Error: Please provide either --input_file or --input_text.")
        return

    for result in results:
        print(f"Original Text: {result['original_text']}")
        print(f"Token Count: {result['token_count']}")
        print("Suggestions:")
        for suggestion in result['suggestions']:
            print(f"- {suggestion}")
        print("---")

if __name__ == "__main__":
    main()