Hallucination Corrector

What It Does

Installation

pip install wikipedia-api nltk pytest

python -c "import nltk; nltk.download('punkt')"

Usage

python hallucination_corrector.py "This is a hallucinated sentence."

{
    "original_text": "This is a hallucinated sentence.",
    "flagged_hallucinations": ["This is a hallucinated sentence."],
    "suggested_corrections": [
        {
            "sentence": "This is a hallucinated sentence.",
            "suggestion": "No reliable information found in the allowed sources."
        }
    ]
}

Source Code

import argparse
import re
from typing import List, Dict, Optional

import wikipediaapi
from nltk.tokenize import sent_tokenize
import nltk

# Ensure NLTK resources are downloaded
try:
    nltk.data.find('tokenizers/punkt')
except LookupError:
    nltk.download('punkt')

def correct_hallucinations(text: str, sources: Optional[List[str]] = None) -> Dict:
    """
    Identifies potential hallucinations in the given text and suggests corrections.

    Args:
        text (str): The LLM-generated output to analyze.
        sources (Optional[List[str]]): List of allowed sources for validation (e.g., ['Wikipedia']).

    Returns:
        Dict: A dictionary containing the original text, flagged hallucinations, and suggested corrections.
    """
    if not text.strip():
        return {
            "original_text": text,
            "flagged_hallucinations": [],
            "suggested_corrections": []
        }

    wiki_wiki = wikipediaapi.Wikipedia('en')
    sentences = sent_tokenize(text)
    flagged_hallucinations = []
    suggested_corrections = []

    for sentence in sentences:
        # Simplified logic for testing purposes
        page = wiki_wiki.page(sentence[:50])  # Use first 50 characters as a mock title

        if not page.exists():
            flagged_hallucinations.append(sentence)
            suggested_corrections.append({
                "sentence": sentence,
                "suggestion": "No reliable information found in the allowed sources."
            })
        else:
            suggested_corrections.append({
                "sentence": sentence,
                "suggestion": f"See Wikipedia article: {page.fullurl}"
            })

    return {
        "original_text": text,
        "flagged_hallucinations": flagged_hallucinations,
        "suggested_corrections": suggested_corrections
    }

def main():
    parser = argparse.ArgumentParser(description="Hallucination Corrector: Identify and correct hallucinations in LLM outputs.")
    parser.add_argument("text", type=str, help="The LLM-generated text to analyze.")
    parser.add_argument("--sources", nargs="*", default=["Wikipedia"], help="List of allowed sources for validation.")

    args = parser.parse_args()

    result = correct_hallucinations(args.text, args.sources)
    print(result)

if __name__ == "__main__":
    main()