fallback

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.10
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# download spacy model and nltk resources at build time
+RUN python -m spacy download en_core_web_sm || true
+RUN python - <<'PY'
+from text_humanizer import download_nltk_resources
+download_nltk_resources()
+PY
+
+EXPOSE 7860
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -1,209 +1,45 @@
-# For Hugging Face Spaces - this is the main app file
-import gradio as gr
-import time
 import os
+from fastapi import FastAPI, Header, HTTPException, Depends
+from pydantic import BaseModel
+from text_humanizer import TextHumanizer, download_nltk_resources
+import spacy
 
-# Import our humanizer
-from text_humanizer import AITextHumanizer
+API_KEY = os.environ.get("API_KEY", "dev-key")
+PORT = int(os.environ.get("PORT", 7860))
 
-# Initialize the humanizer
-print("🚀 Loading AI Text Humanizer for Hugging Face Spaces...")
-try:
-    humanizer = AITextHumanizer()
-    print("✅ Humanizer loaded successfully!")
-except Exception as e:
-    print(f"❌ Error loading humanizer: {e}")
-    humanizer = None
+app = FastAPI()
+humanizer = None
 
-def humanize_text_hf(text, style, intensity):
-    """
-    Hugging Face Spaces interface function for text humanization
-    """
-    if not text.strip():
-        return "⚠️ Please enter some text to humanize.", "", 0.0, "No changes made", 0.0
-    
-    if humanizer is None:
-        return "❌ Error: Humanizer not loaded properly.", "", 0.0, "System error", 0.0
-    
+class HumanizeReq(BaseModel):
+    text: str
+    use_passive: bool = False
+    use_synonyms: bool = False
+
+def verify_key(x_api_key: str = Header(None)):
+    if x_api_key != API_KEY:
+        raise HTTPException(status_code=403, detail="Forbidden")
+    return True
+
+@app.get("/")
+def greet_json():
+    return {"Hello": "World!"}
+
+@app.on_event("startup")
+def startup():
+    # ensure NLTK resources and spacy model are available at runtime
+    download_nltk_resources()
     try:
-        start_time = time.time()
-        
-        # Humanize the text
-        result = humanizer.humanize_text(
-            text=text,
-            style=style.lower(),
-            intensity=intensity
-        )
-        
-        processing_time = (time.time() - start_time) * 1000
-        
-        changes_text = ", ".join(result["changes_made"]) if result["changes_made"] else "No significant changes made"
-        
-        return (
-            result["humanized_text"],
-            f"**📊 Processing Results:**\n- **Similarity Score:** {result['similarity_score']:.3f}\n- **Processing Time:** {processing_time:.1f}ms\n- **Style:** {result['style'].title()}\n- **Intensity:** {result['intensity']}\n\n**🔄 Changes Made:** {changes_text}",
-            result["similarity_score"],
-            changes_text,
-            processing_time
-        )
-        
-    except Exception as e:
-        return f"❌ Error processing text: {str(e)}", "", 0.0, "Processing error", 0.0
+        spacy.load("en_core_web_sm")
+    except OSError:
+        import spacy.cli
+        spacy.cli.download("en_core_web_sm")
+    global humanizer
+    humanizer = TextHumanizer()
 
-# Create the Hugging Face Spaces interface
-with gr.Blocks(
-    title="🤖➡️👤 AI Text Humanizer",
-    theme=gr.themes.Soft(),
-    css="""
-        .main-header {
-            text-align: center;
-            background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);
-            color: white;
-            padding: 20px;
-            border-radius: 10px;
-            margin-bottom: 20px;
-        }
-        .stats-box {
-            background: #f8f9fa;
-            padding: 15px;
-            border-radius: 8px;
-            border-left: 4px solid #667eea;
-        }
-    """
-) as iface:
-    
-    gr.HTML("""
-        <div class="main-header">
-            <h1>🤖➡️👤 AI Text Humanizer</h1>
-            <p>Transform AI-generated text to sound more natural and human-like</p>
-            <p><em>Powered by advanced NLP techniques and transformers</em></p>
-        </div>
-    """)
-    
-    with gr.Tab("🎯 Humanize Text"):
-        with gr.Row():
-            with gr.Column(scale=1):
-                gr.HTML("<h3>📝 Input</h3>")
-                
-                input_text = gr.Textbox(
-                    label="Text to Humanize",
-                    placeholder="Paste your AI-generated text here...\n\nExample: Furthermore, it is important to note that artificial intelligence systems demonstrate significant capabilities...",
-                    lines=10,
-                    max_lines=20
-                )
-                
-                with gr.Row():
-                    style_dropdown = gr.Dropdown(
-                        choices=["Natural", "Casual", "Conversational"],
-                        value="Natural",
-                        label="🎨 Humanization Style"
-                    )
-                    
-                    intensity_slider = gr.Slider(
-                        minimum=0.1,
-                        maximum=1.0,
-                        value=0.7,
-                        step=0.1,
-                        label="⚡ Intensity Level"
-                    )
-                
-                humanize_btn = gr.Button(
-                    "🚀 Humanize Text", 
-                    variant="primary",
-                    size="lg"
-                )
-            
-            with gr.Column(scale=1):
-                gr.HTML("<h3>✨ Output</h3>")
-                
-                output_text = gr.Textbox(
-                    label="Humanized Text",
-                    lines=10,
-                    max_lines=20,
-                    show_copy_button=True
-                )
-                
-                stats_output = gr.Markdown(
-                    label="📊 Processing Statistics",
-                    value="Results will appear here after processing..."
-                )
-    
-    with gr.Tab("📊 Examples & Guide"):
-        gr.HTML("<h3>💡 Try These Examples</h3>")
-        
-        # Examples
-        examples = gr.Examples(
-            examples=[
-                [
-                    "Furthermore, it is important to note that artificial intelligence systems demonstrate significant capabilities in natural language processing tasks. Subsequently, these systems can analyze and generate text with remarkable accuracy. Nevertheless, it is crucial to understand that human oversight remains essential for optimal performance.",
-                    "Conversational",
-                    0.8
-                ],
-                [
-                    "The implementation of this comprehensive solution will facilitate the optimization of business processes and operational workflows. Moreover, it will demonstrate substantial improvements in efficiency metrics while maintaining quality standards.",
-                    "Natural", 
-                    0.6
-                ],
-                [
-                    "In conclusion, the systematic analysis reveals that the proposed methodology demonstrates significant potential for enhancing performance indicators. Additionally, the structured approach ensures optimal resource utilization.",
-                    "Casual",
-                    0.7
-                ]
-            ],
-            inputs=[input_text, style_dropdown, intensity_slider],
-            outputs=[output_text, stats_output],
-            fn=humanize_text_hf,
-            cache_examples=False
-        )
-        
-        gr.HTML("""
-            <div style="margin-top: 30px;">
-                <h3>🎯 How It Works</h3>
-                <div class="stats-box">
-                    <h4>🔧 Transformation Techniques:</h4>
-                    <ul>
-                        <li><strong>Smart Word Replacement:</strong> formal words → casual alternatives</li>
-                        <li><strong>Contraction Addition:</strong> "do not" → "don't", "it is" → "it's"</li>
-                        <li><strong>AI Transition Removal:</strong> removes robotic transition phrases</li>
-                        <li><strong>Sentence Restructuring:</strong> varies length and structure</li>
-                        <li><strong>Natural Imperfections:</strong> adds human-like variations</li>
-                        <li><strong>Context-Aware Paraphrasing:</strong> maintains meaning while improving flow</li>
-                    </ul>
-                </div>
-                
-                <div class="stats-box" style="margin-top: 15px;">
-                    <h4>🎨 Style Guide:</h4>
-                    <ul>
-                        <li><strong>Natural (0.5-0.7):</strong> Professional content with human touch</li>
-                        <li><strong>Casual (0.6-0.8):</strong> Blog posts, articles, informal content</li>
-                        <li><strong>Conversational (0.7-1.0):</strong> Social media, very informal text</li>
-                    </ul>
-                </div>
-                
-                <div class="stats-box" style="margin-top: 15px;">
-                    <h4>⚡ Performance:</h4>
-                    <ul>
-                        <li><strong>Similarity Preservation:</strong> Maintains 85-95% semantic similarity</li>
-                        <li><strong>Processing Speed:</strong> ~500ms for typical paragraphs</li>
-                        <li><strong>Quality:</strong> Advanced NLP models ensure high-quality output</li>
-                    </ul>
-                </div>
-            </div>
-        """)
-    
-    # Event handlers
-    humanize_btn.click(
-        fn=humanize_text_hf,
-        inputs=[input_text, style_dropdown, intensity_slider],
-        outputs=[output_text, stats_output]
-    )
+@app.post("/humanize")
+def humanize(req: HumanizeReq, _=Depends(verify_key)):
+    return {"humanized": humanizer.humanize_text(req.text, req.use_passive, req.use_synonyms)}
 
-# Launch for Hugging Face Spaces
-if __name__ == "__main__":
-    print("🌐 Launching AI Text Humanizer on Hugging Face Spaces...")
-    iface.launch(
-        share=False,  # HF Spaces handles sharing
-        server_name="0.0.0.0",
-        server_port=7860,
-        show_error=True
-    )
+# if __name__ == "__main__":
+#     import uvicorn
+#     uvicorn.run(app, host="0.0.0.0", port=PORT)

requirements.txt

@@ -1,15 +1,5 @@
-fastapi==0.104.1
-uvicorn[standard]==0.24.0
-gradio==4.7.1
-transformers==4.35.0
-torch==2.1.0
-sentence-transformers==2.2.2
-nltk==3.8.1
-spacy>=3.7.0
-pydantic==2.5.0
-numpy==1.25.2
-pandas==2.1.3
-redis==5.0.1
-python-multipart==0.0.6
-aiofiles==23.2.1
-requests==2.31.0
+fastapi
+uvicorn[standard]
+spacy
+nltk
+sentence-transformers

text_humanizer.py

@@ -1,390 +1,200 @@
-import re
+import ssl
 import random
-import nltk
-from typing import List, Dict, Optional
-from sentence_transformers import SentenceTransformer
-import numpy as np
-from transformers import pipeline
+import warnings
 
-# Download required NLTK data
-try:
-    nltk.data.find('tokenizers/punkt')
-except LookupError:
-    nltk.download('punkt')
+import nltk
+import spacy
+from nltk.tokenize import word_tokenize
+from nltk.corpus import wordnet
+from sentence_transformers import SentenceTransformer, util
 
-try:
-    nltk.data.find('corpora/wordnet')
-except LookupError:
-    nltk.download('wordnet')
+warnings.filterwarnings("ignore", category=FutureWarning)
 
-try:
-    nltk.data.find('corpora/omw-1.4')
-except LookupError:
-    nltk.download('omw-1.4')
+NLP_GLOBAL = spacy.load("en_core_web_sm")
 
-from nltk.tokenize import sent_tokenize, word_tokenize
-from nltk.corpus import wordnet
+def download_nltk_resources():
+    """
+    Download required NLTK resources if not already installed.
+    """
+    try:
+        _create_unverified_https_context = ssl._create_unverified_context
+    except AttributeError:
+        pass
+    else:
+        ssl._create_default_https_context = _create_unverified_https_context
 
-class AITextHumanizer:
-    def __init__(self):
-        """Initialize the text humanizer with necessary models and data"""
-        print("Loading models...")
-        
-        # Load sentence transformer for semantic similarity
+    resources = ['punkt', 'averaged_perceptron_tagger', 'punkt_tab','wordnet','averaged_perceptron_tagger_eng']
+    for resource in resources:
         try:
-            self.similarity_model = SentenceTransformer('all-MiniLM-L6-v2')
+            nltk.download(resource, quiet=True)
         except Exception as e:
-            print(f"Warning: Could not load similarity model: {e}")
-            self.similarity_model = None
-        
-        # Initialize paraphrasing pipeline
-        try:
-            self.paraphraser = pipeline("text2text-generation", 
-                                      model="google/flan-t5-small", 
-                                      max_length=512)
-        except Exception as e:
-            print(f"Warning: Could not load paraphrasing model: {e}")
-            self.paraphraser = None
-        
-        # Formal to casual word mappings
-        self.formal_to_casual = {
-            "utilize": "use",
-            "demonstrate": "show",
-            "facilitate": "help",
-            "implement": "do",
-            "consequently": "so",
-            "therefore": "so",
-            "nevertheless": "but",
-            "furthermore": "also",
-            "moreover": "also",
-            "subsequently": "then",
-            "accordingly": "so",
-            "regarding": "about",
-            "concerning": "about",
-            "pertaining": "about",
-            "approximately": "about",
-            "endeavor": "try",
-            "commence": "start",
-            "terminate": "end",
-            "obtain": "get",
-            "purchase": "buy",
-            "examine": "look at",
-            "analyze": "study",
-            "construct": "build",
-            "establish": "set up",
-            "magnitude": "size",
-            "comprehensive": "complete",
-            "significant": "big",
-            "substantial": "large",
-            "optimal": "best",
-            "sufficient": "enough",
-            "prior to": "before",
-            "in order to": "to",
-            "due to the fact that": "because",
-            "at this point in time": "now",
-            "in the event that": "if",
-        }
-        
-        # Contractions mapping
-        self.contractions = {
-            "do not": "don't",
-            "does not": "doesn't",
-            "did not": "didn't",
-            "will not": "won't",
-            "would not": "wouldn't",
-            "should not": "shouldn't",
-            "could not": "couldn't",
-            "cannot": "can't",
-            "is not": "isn't",
-            "are not": "aren't",
-            "was not": "wasn't",
-            "were not": "weren't",
-            "have not": "haven't",
-            "has not": "hasn't",
-            "had not": "hadn't",
-            "I am": "I'm",
-            "you are": "you're",
-            "he is": "he's",
-            "she is": "she's",
-            "it is": "it's",
-            "we are": "we're",
-            "they are": "they're",
-            "I have": "I've",
-            "you have": "you've",
-            "we have": "we've",
-            "they have": "they've",
-            "I will": "I'll",
-            "you will": "you'll",
-            "he will": "he'll",
-            "she will": "she'll",
-            "it will": "it'll",
-            "we will": "we'll",
-            "they will": "they'll",
-        }
-        
-        # Transition words that make text sound more AI-like
-        self.ai_transition_words = [
-            "Furthermore,", "Moreover,", "Additionally,", "Subsequently,",
-            "Consequently,", "Therefore,", "Nevertheless,", "However,",
-            "In conclusion,", "To summarize,", "In summary,", "Overall,",
-            "It is important to note that", "It should be emphasized that",
-            "It is worth mentioning that", "It is crucial to understand that"
-        ]
-        
-        # Natural alternatives
-        self.natural_transitions = [
-            "Also,", "Plus,", "And,", "Then,", "So,", "But,", "Still,",
-            "Anyway,", "By the way,", "Actually,", "Basically,",
-            "Look,", "Listen,", "Here's the thing:", "The point is,",
-            "What's more,", "On top of that,", "Another thing,",
+            print(f"Error downloading {resource}: {str(e)}")
+
+
+# This class  contains methods to humanize academic text, such as improving readability or
+# simplifying complex language.
+class TextHumanizer:
+    """
+    Transforms text into a more formal (academic) style:
+      - Expands contractions
+      - Adds academic transitions
+      - Optionally converts some sentences to passive voice
+      - Optionally replaces words with synonyms for more formality
+    """
+
+    def __init__(
+        self,
+        model_name='paraphrase-MiniLM-L6-v2',
+        p_passive=0.2,
+        p_synonym_replacement=0.3,
+        p_academic_transition=0.3,
+        seed=None
+    ):
+        if seed is not None:
+            random.seed(seed)
+
+        self.nlp = spacy.load("en_core_web_sm")
+        self.model = SentenceTransformer(model_name)
+
+        # Transformation probabilities
+        self.p_passive = p_passive
+        self.p_synonym_replacement = p_synonym_replacement
+        self.p_academic_transition = p_academic_transition
+
+        # Common academic transitions
+        self.academic_transitions = [
+            "Moreover,", "Additionally,", "Furthermore,", "Hence,", 
+            "Therefore,", "Consequently,", "Nonetheless,", "Nevertheless,"
         ]
-        
-        print("Humanizer initialized successfully!")
-    
-    def add_contractions(self, text: str) -> str:
-        """Add contractions to make text sound more natural"""
-        for formal, casual in self.contractions.items():
-            # Case insensitive replacement but preserve original case
-            pattern = re.compile(re.escape(formal), re.IGNORECASE)
-            text = pattern.sub(casual, text)
-        return text
-    
-    def replace_formal_words(self, text: str, replacement_rate: float = 0.7) -> str:
-        """Replace formal words with casual alternatives"""
-        words = word_tokenize(text)
-        
-        for i, word in enumerate(words):
-            word_lower = word.lower()
-            if word_lower in self.formal_to_casual and random.random() < replacement_rate:
-                # Preserve original case
-                if word.isupper():
-                    words[i] = self.formal_to_casual[word_lower].upper()
-                elif word.istitle():
-                    words[i] = self.formal_to_casual[word_lower].title()
-                else:
-                    words[i] = self.formal_to_casual[word_lower]
-        
-        # Reconstruct text with proper spacing
-        result = ""
-        for i, word in enumerate(words):
-            if i > 0 and word not in ".,!?;:":
-                result += " "
-            result += word
-        
-        return result
-    
-    def vary_sentence_structure(self, text: str) -> str:
-        """Vary sentence structure to sound more natural"""
-        sentences = sent_tokenize(text)
-        varied_sentences = []
-        
-        for sentence in sentences:
-            # Sometimes start sentences with connecting words
-            if random.random() < 0.3:
-                connectors = ["Well,", "So,", "Now,", "Look,", "Actually,", "Basically,"]
-                if not any(sentence.startswith(word) for word in connectors):
-                    sentence = random.choice(connectors) + " " + sentence.lower()
-            
-            # Occasionally break long sentences
-            if len(sentence.split()) > 20 and random.random() < 0.4:
-                words = sentence.split()
-                mid_point = len(words) // 2
-                # Find a natural break point near the middle
-                for i in range(mid_point - 3, min(mid_point + 3, len(words))):
-                    if words[i] in [',', 'and', 'but', 'or', 'so']:
-                        sentence1 = ' '.join(words[:i+1])
-                        sentence2 = ' '.join(words[i+1:])
-                        if sentence2:
-                            sentence2 = sentence2[0].upper() + sentence2[1:]
-                            varied_sentences.append(sentence1)
-                            sentence = sentence2
-                        break
-            
-            varied_sentences.append(sentence)
-        
-        return ' '.join(varied_sentences)
-    
-    def replace_ai_transitions(self, text: str) -> str:
-        """Replace AI-like transition words with natural alternatives"""
-        for ai_word in self.ai_transition_words:
-            if ai_word in text:
-                natural_replacement = random.choice(self.natural_transitions)
-                text = text.replace(ai_word, natural_replacement)
-        return text
-    
-    def add_natural_imperfections(self, text: str, imperfection_rate: float = 0.1) -> str:
-        """Add subtle imperfections to make text more human-like"""
-        sentences = sent_tokenize(text)
-        modified_sentences = []
-        
-        for sentence in sentences:
-            # Occasionally start with lowercase after punctuation (casual style)
-            if random.random() < imperfection_rate:
-                words = sentence.split()
-                if len(words) > 1 and words[0].lower() in ['and', 'but', 'or', 'so']:
-                    sentence = words[0].lower() + ' ' + ' '.join(words[1:])
-            
-            # Sometimes use informal punctuation
-            if random.random() < imperfection_rate:
-                if sentence.endswith('.'):
-                    sentence = sentence[:-1]  # Remove period occasionally
-                elif not sentence.endswith(('.', '!', '?')):
-                    if random.random() < 0.5:
-                        sentence += '.'
-            
-            modified_sentences.append(sentence)
-        
-        return ' '.join(modified_sentences)
-    
-    def paraphrase_segments(self, text: str, paraphrase_rate: float = 0.3) -> str:
-        """Paraphrase some segments using the transformer model"""
-        if not self.paraphraser:
-            return text
-        
-        sentences = sent_tokenize(text)
-        paraphrased_sentences = []
-        
-        for sentence in sentences:
-            if random.random() < paraphrase_rate and len(sentence.split()) > 5:
-                try:
-                    # Create paraphrase prompt
-                    prompt = f"Rewrite this sentence in a more natural, conversational way: {sentence}"
-                    
-                    result = self.paraphraser(prompt, max_length=100, num_return_sequences=1)
-                    paraphrased = result[0]['generated_text']
-                    
-                    # Clean up the result
-                    paraphrased = paraphrased.replace(prompt, '').strip()
-                    if paraphrased and len(paraphrased) > 10:
-                        paraphrased_sentences.append(paraphrased)
+
+    def humanize_text(self, text, use_passive=False, use_synonyms=False):
+        doc = self.nlp(text)
+        transformed_sentences = []
+
+        for sent in doc.sents:
+            sentence_str = sent.text.strip()
+
+            # 1. Expand contractions
+            sentence_str = self.expand_contractions(sentence_str)
+
+            # 2. Possibly add academic transitions
+            # if random.random() < self.p_academic_transition:
+            #     sentence_str = self.add_academic_transitions(sentence_str)
+
+            # 3. Optionally convert to passive
+            if use_passive and random.random() < self.p_passive:
+                sentence_str = self.convert_to_passive(sentence_str)
+
+            # 4. Optionally replace words with synonyms
+            if use_synonyms and random.random() < self.p_synonym_replacement:
+                sentence_str = self.replace_with_synonyms(sentence_str)
+
+            transformed_sentences.append(sentence_str)
+
+        return ' '.join(transformed_sentences)
+
+    def expand_contractions(self, sentence):
+        contraction_map = {
+            "n't": " not", "'re": " are", "'s": " is", "'ll": " will",
+            "'ve": " have", "'d": " would", "'m": " am"
+        }
+        tokens = word_tokenize(sentence)
+        expanded_tokens = []
+        for token in tokens:
+            lower_token = token.lower()
+            replaced = False
+            for contraction, expansion in contraction_map.items():
+                if contraction in lower_token and lower_token.endswith(contraction):
+                    new_token = lower_token.replace(contraction, expansion)
+                    if token[0].isupper():
+                        new_token = new_token.capitalize()
+                    expanded_tokens.append(new_token)
+                    replaced = True
+                    break
+            if not replaced:
+                expanded_tokens.append(token)
+
+        return ' '.join(expanded_tokens)
+
+    def add_academic_transitions(self, sentence):
+        transition = random.choice(self.academic_transitions)
+        return f"{transition} {sentence}"
+
+    def convert_to_passive(self, sentence):
+        doc = self.nlp(sentence)
+        subj_tokens = [t for t in doc if t.dep_ == 'nsubj' and t.head.dep_ == 'ROOT']
+        dobj_tokens = [t for t in doc if t.dep_ == 'dobj']
+
+        if subj_tokens and dobj_tokens:
+            subject = subj_tokens[0]
+            dobj = dobj_tokens[0]
+            verb = subject.head
+            if subject.i < verb.i < dobj.i:
+                passive_str = f"{dobj.text} {verb.lemma_} by {subject.text}"
+                original_str = ' '.join(token.text for token in doc)
+                chunk = f"{subject.text} {verb.text} {dobj.text}"
+                if chunk in original_str:
+                    sentence = original_str.replace(chunk, passive_str)
+        return sentence
+
+    def replace_with_synonyms(self, sentence):
+        tokens = word_tokenize(sentence)
+        pos_tags = nltk.pos_tag(tokens)
+
+        new_tokens = []
+        for (word, pos) in pos_tags:
+            if pos.startswith(('J', 'N', 'V', 'R')) and wordnet.synsets(word):
+                if random.random() < 0.5:
+                    synonyms = self._get_synonyms(word, pos)
+                    if synonyms:
+                        best_synonym = self._select_closest_synonym(word, synonyms)
+                        new_tokens.append(best_synonym if best_synonym else word)
                     else:
-                        paraphrased_sentences.append(sentence)
-                except Exception as e:
-                    print(f"Paraphrasing failed: {e}")
-                    paraphrased_sentences.append(sentence)
+                        new_tokens.append(word)
+                else:
+                    new_tokens.append(word)
             else:
-                paraphrased_sentences.append(sentence)
-        
-        return ' '.join(paraphrased_sentences)
-    
-    def calculate_similarity(self, text1: str, text2: str) -> float:
-        """Calculate semantic similarity between original and humanized text"""
-        if not self.similarity_model:
-            return 0.85  # Return reasonable default if model not available
-        
-        try:
-            embeddings1 = self.similarity_model.encode([text1])
-            embeddings2 = self.similarity_model.encode([text2])
-            similarity = np.dot(embeddings1[0], embeddings2[0]) / (
-                np.linalg.norm(embeddings1[0]) * np.linalg.norm(embeddings2[0])
-            )
-            return float(similarity)
-        except Exception as e:
-            print(f"Similarity calculation failed: {e}")
-            return 0.85
-    
-    def humanize_text(self, 
-                     text: str, 
-                     style: str = "natural",
-                     intensity: float = 0.7) -> Dict:
-        """
-        Main humanization function
-        
-        Args:
-            text: Input text to humanize
-            style: Style of humanization ('natural', 'casual', 'conversational')
-            intensity: Intensity of humanization (0.0 to 1.0)
-        
-        Returns:
-            Dictionary with humanized text and metadata
-        """
-        if not text.strip():
-            return {
-                "original_text": text,
-                "humanized_text": text,
-                "similarity_score": 1.0,
-                "changes_made": []
-            }
-        
-        changes_made = []
-        humanized_text = text
-        
-        # Apply transformations based on intensity
-        if intensity > 0.2:
-            # Replace formal words
-            before_formal = humanized_text
-            humanized_text = self.replace_formal_words(humanized_text, intensity * 0.7)
-            if humanized_text != before_formal:
-                changes_made.append("Replaced formal words with casual alternatives")
-        
-        if intensity > 0.3:
-            # Add contractions
-            before_contractions = humanized_text
-            humanized_text = self.add_contractions(humanized_text)
-            if humanized_text != before_contractions:
-                changes_made.append("Added contractions")
-        
-        if intensity > 0.4:
-            # Replace AI-like transitions
-            before_transitions = humanized_text
-            humanized_text = self.replace_ai_transitions(humanized_text)
-            if humanized_text != before_transitions:
-                changes_made.append("Replaced AI-like transition words")
-        
-        if intensity > 0.5:
-            # Vary sentence structure
-            before_structure = humanized_text
-            humanized_text = self.vary_sentence_structure(humanized_text)
-            if humanized_text != before_structure:
-                changes_made.append("Varied sentence structure")
-        
-        if intensity > 0.6 and style in ["casual", "conversational"]:
-            # Add natural imperfections
-            before_imperfections = humanized_text
-            humanized_text = self.add_natural_imperfections(humanized_text, intensity * 0.2)
-            if humanized_text != before_imperfections:
-                changes_made.append("Added natural imperfections")
-        
-        if intensity > 0.7:
-            # Paraphrase some segments
-            before_paraphrase = humanized_text
-            humanized_text = self.paraphrase_segments(humanized_text, intensity * 0.4)
-            if humanized_text != before_paraphrase:
-                changes_made.append("Paraphrased some segments")
-        
-        # Calculate similarity
-        similarity_score = self.calculate_similarity(text, humanized_text)
-        
-        return {
-            "original_text": text,
-            "humanized_text": humanized_text,
-            "similarity_score": similarity_score,
-            "changes_made": changes_made,
-            "style": style,
-            "intensity": intensity
-        }
+                new_tokens.append(word)
 
-# Test the humanizer
-if __name__ == "__main__":
-    humanizer = AITextHumanizer()
-    
-    # Test text
-    test_text = """
-    Furthermore, it is important to note that artificial intelligence systems demonstrate 
-    significant capabilities in natural language processing tasks. Subsequently, these 
-    systems can analyze and generate text with remarkable accuracy. Nevertheless, it is 
-    crucial to understand that human oversight remains essential for optimal performance. 
-    Therefore, organizations should implement comprehensive strategies to utilize these 
-    technologies effectively while maintaining quality standards.
-    """
-    
-    print("Original Text:")
-    print(test_text)
-    print("\n" + "="*50 + "\n")
-    
-    result = humanizer.humanize_text(test_text, style="conversational", intensity=0.8)
-    
-    print("Humanized Text:")
-    print(result["humanized_text"])
-    print(f"\nSimilarity Score: {result['similarity_score']:.3f}")
-    print(f"Changes Made: {', '.join(result['changes_made'])}")
+        # Join cleanly with punctuation fix
+        sentence = " ".join(new_tokens)
+        sentence = (
+            sentence.replace(" ,", ",")
+            .replace(" .", ".")
+            .replace(" !", "!")
+            .replace(" ?", "?")
+            .replace(" :", ":")
+            .replace(" '", "'")
+        )
+        return sentence
+
+    def _get_synonyms(self, word, pos):
+        wn_pos = None
+        if pos.startswith('J'):
+            wn_pos = wordnet.ADJ
+        elif pos.startswith('N'):
+            wn_pos = wordnet.NOUN
+        elif pos.startswith('R'):
+            wn_pos = wordnet.ADV
+        elif pos.startswith('V'):
+            wn_pos = wordnet.VERB
+
+        synonyms = set()
+        for syn in wordnet.synsets(word, pos=wn_pos):
+            for lemma in syn.lemmas():
+                lemma_name = lemma.name().replace('_', ' ')
+                if lemma_name.lower() != word.lower():
+                    synonyms.add(lemma_name)
+        return list(synonyms)
+
+    def _select_closest_synonym(self, original_word, synonyms):
+        if not synonyms:
+            return None
+        original_emb = self.model.encode(original_word, convert_to_tensor=True)
+        synonym_embs = self.model.encode(synonyms, convert_to_tensor=True)
+        cos_scores = util.cos_sim(original_emb, synonym_embs)[0]
+        max_score_index = cos_scores.argmax().item()
+        max_score = cos_scores[max_score_index].item()
+        if max_score >= 0.5:
+            return synonyms[max_score_index]
+        return None