Batch LLM Text Auditor

What It Does

Installation

git clone <repository_url>
   cd batch_llm_text_auditor

pip install -r requirements.txt

Usage

python batch_llm_text_auditor.py --input_dir ./texts --output ./audit_report.csv

Source Code

import os
import argparse
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import nltk
from nltk.corpus import stopwords
import string
import json

nltk.download('stopwords', quiet=True)

class BatchLLMTextAuditor:
    def __init__(self, input_dir, output_file, threshold=0.5):
        self.input_dir = input_dir
        self.output_file = output_file
        self.threshold = threshold
        self.stop_words = set(stopwords.words('english'))
        self.model = RandomForestClassifier()
        self.vectorizer = TfidfVectorizer()

    def preprocess_text(self, text):
        text = text.lower()
        text = ''.join([char for char in text if char not in string.punctuation])
        words = text.split()
        words = [word for word in words if word not in self.stop_words]
        return ' '.join(words)

    def load_data(self):
        texts = []
        filenames = []
        for root, _, files in os.walk(self.input_dir):
            for file in files:
                if file.endswith('.txt'):
                    file_path = os.path.join(root, file)
                    with open(file_path, 'r', encoding='utf-8') as f:
                        texts.append(f.read())
                        filenames.append(file)
        return filenames, texts

    def train_model(self):
        # Simulating a pre-trained model for demonstration purposes
        dummy_texts = ["This is a human-generated text.", "This text is generated by an AI."]
        dummy_labels = [0, 1]  # 0: Human, 1: LLM
        preprocessed_dummy_texts = [self.preprocess_text(text) for text in dummy_texts]
        features = self.vectorizer.fit_transform(preprocessed_dummy_texts)
        self.model.fit(features, dummy_labels)

    def classify_texts(self, texts):
        preprocessed_texts = [self.preprocess_text(text) for text in texts]
        features = self.vectorizer.transform(preprocessed_texts)
        probabilities = self.model.predict_proba(features)[:, 1]
        return probabilities

    def generate_report(self, filenames, texts, probabilities):
        data = {
            'Filename': filenames,
            'Text': texts,
            'LLM_Probability': probabilities
        }
        df = pd.DataFrame(data)
        df['LLM_Prediction'] = df['LLM_Probability'] > self.threshold
        df.to_csv(self.output_file, index=False)

    def run(self):
        filenames, texts = self.load_data()
        if not texts:
            print("No text files found in the input directory.")
            return

        self.train_model()

        probabilities = self.classify_texts(texts)

        self.generate_report(filenames, texts, probabilities)
        print(f"Report generated: {self.output_file}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Batch LLM Text Auditor")
    parser.add_argument("--input_dir", required=True, help="Directory containing text files to audit")
    parser.add_argument("--output", required=True, help="Output CSV file for the results")
    args = parser.parse_args()

    auditor = BatchLLMTextAuditor(input_dir=args.input_dir, output_file=args.output)
    auditor.run()