| """
|
| Unit Tests for Text Preprocessing Module.
|
|
|
| Tests text cleaning, normalization, and utility functions.
|
| """
|
|
|
| import pytest
|
|
|
| from app.utils.preprocessing import (
|
| clean_text,
|
| normalize_text,
|
| convert_devanagari_digits,
|
| truncate_text,
|
| remove_urls,
|
| extract_numbers,
|
| mask_sensitive_data,
|
| )
|
|
|
|
|
| class TestCleanText:
|
| """Tests for clean_text function."""
|
|
|
| def test_empty_string(self):
|
| """Test empty string returns empty string."""
|
| assert clean_text("") == ""
|
|
|
| def test_none_returns_empty(self):
|
| """Test None or falsy value returns empty string."""
|
| assert clean_text(None) == ""
|
|
|
| def test_removes_extra_whitespace(self):
|
| """Test extra whitespace is normalized."""
|
| text = "Hello world here"
|
| result = clean_text(text)
|
| assert result == "Hello world here"
|
|
|
| def test_removes_leading_trailing_whitespace(self):
|
| """Test leading/trailing whitespace is stripped."""
|
| text = " Hello world "
|
| result = clean_text(text)
|
| assert result == "Hello world"
|
|
|
| def test_removes_control_characters(self):
|
| """Test control characters are removed."""
|
| text = "Hello\x00\x07world"
|
| result = clean_text(text)
|
| assert "\x00" not in result
|
| assert "\x07" not in result
|
| assert "Hello" in result
|
| assert "world" in result
|
|
|
| def test_preserves_normal_text(self):
|
| """Test normal text is preserved."""
|
| text = "Hello, how are you?"
|
| result = clean_text(text)
|
| assert result == text
|
|
|
| def test_normalizes_newlines_and_tabs(self):
|
| """Test newlines and tabs are normalized to spaces."""
|
| text = "Hello\nworld\there"
|
| result = clean_text(text)
|
| assert result == "Hello world here"
|
|
|
| def test_handles_unicode(self):
|
| """Test Unicode text is preserved."""
|
| text = "नमस्ते दुनिया"
|
| result = clean_text(text)
|
| assert result == text
|
|
|
|
|
| class TestNormalizeText:
|
| """Tests for normalize_text function."""
|
|
|
| def test_basic_normalization(self):
|
| """Test basic text normalization."""
|
| text = " Hello world "
|
| result = normalize_text(text)
|
| assert result == "Hello world"
|
|
|
| def test_lowercase_option(self):
|
| """Test lowercase option."""
|
| text = "Hello WORLD"
|
| result = normalize_text(text, lowercase=True)
|
| assert result == "hello world"
|
|
|
| def test_without_lowercase(self):
|
| """Test preserves case by default."""
|
| text = "Hello WORLD"
|
| result = normalize_text(text, lowercase=False)
|
| assert result == "Hello WORLD"
|
|
|
| def test_converts_devanagari_digits(self):
|
| """Test Devanagari digits are converted."""
|
| text = "Amount: ५०००"
|
| result = normalize_text(text)
|
| assert "5000" in result
|
|
|
|
|
| class TestConvertDevanagariDigits:
|
| """Tests for convert_devanagari_digits function."""
|
|
|
| def test_converts_all_digits(self):
|
| """Test all Devanagari digits are converted."""
|
| text = "०१२३४५६७८९"
|
| result = convert_devanagari_digits(text)
|
| assert result == "0123456789"
|
|
|
| def test_preserves_latin_digits(self):
|
| """Test Latin digits are preserved."""
|
| text = "123456"
|
| result = convert_devanagari_digits(text)
|
| assert result == "123456"
|
|
|
| def test_mixed_digits(self):
|
| """Test mixed Devanagari and Latin digits."""
|
| text = "Phone: ९८76543२१०"
|
| result = convert_devanagari_digits(text)
|
| assert result == "Phone: 9876543210"
|
|
|
| def test_preserves_non_digit_text(self):
|
| """Test non-digit text is preserved."""
|
| text = "नमस्ते"
|
| result = convert_devanagari_digits(text)
|
| assert result == "नमस्ते"
|
|
|
| def test_empty_string(self):
|
| """Test empty string returns empty."""
|
| assert convert_devanagari_digits("") == ""
|
|
|
| def test_phone_number_in_hindi(self):
|
| """Test phone number conversion in Hindi context."""
|
| text = "कॉल करें ९८७६५४३२१०"
|
| result = convert_devanagari_digits(text)
|
| assert "9876543210" in result
|
|
|
|
|
| class TestTruncateText:
|
| """Tests for truncate_text function."""
|
|
|
| def test_short_text_unchanged(self):
|
| """Test text shorter than limit is unchanged."""
|
| text = "Hello world"
|
| result = truncate_text(text, max_length=100)
|
| assert result == text
|
|
|
| def test_long_text_truncated(self):
|
| """Test text longer than limit is truncated."""
|
| text = "a" * 100
|
| result = truncate_text(text, max_length=50)
|
| assert len(result) == 50
|
| assert result.endswith("...")
|
|
|
| def test_custom_suffix(self):
|
| """Test custom truncation suffix."""
|
| text = "a" * 100
|
| result = truncate_text(text, max_length=50, suffix="[...]")
|
| assert result.endswith("[...]")
|
|
|
| def test_exact_length(self):
|
| """Test text at exact length is unchanged."""
|
| text = "a" * 50
|
| result = truncate_text(text, max_length=50)
|
| assert result == text
|
|
|
| def test_default_max_length(self):
|
| """Test default max_length is 5000."""
|
| text = "a" * 5000
|
| result = truncate_text(text)
|
| assert len(result) == 5000
|
|
|
|
|
| class TestRemoveUrls:
|
| """Tests for remove_urls function."""
|
|
|
| def test_removes_http_url(self):
|
| """Test HTTP URLs are removed."""
|
| text = "Visit http://example.com for more info"
|
| result = remove_urls(text)
|
| assert "http://example.com" not in result
|
| assert "Visit" in result
|
|
|
| def test_removes_https_url(self):
|
| """Test HTTPS URLs are removed."""
|
| text = "Visit https://secure.example.com for more info"
|
| result = remove_urls(text)
|
| assert "https://secure.example.com" not in result
|
|
|
| def test_removes_multiple_urls(self):
|
| """Test multiple URLs are removed."""
|
| text = "Visit http://one.com and http://two.com"
|
| result = remove_urls(text)
|
| assert "http://one.com" not in result
|
| assert "http://two.com" not in result
|
|
|
| def test_preserves_non_url_text(self):
|
| """Test non-URL text is preserved."""
|
| text = "Hello world, no URLs here"
|
| result = remove_urls(text)
|
| assert result == text
|
|
|
| def test_removes_complex_url(self):
|
| """Test complex URLs with paths are removed."""
|
| text = "Click http://example.com/path/to/page?query=value"
|
| result = remove_urls(text)
|
| assert "http://example.com" not in result
|
|
|
|
|
| class TestExtractNumbers:
|
| """Tests for extract_numbers function."""
|
|
|
| def test_extracts_single_number(self):
|
| """Test extracts single number."""
|
| text = "Amount is 5000"
|
| result = extract_numbers(text)
|
| assert "5000" in result
|
|
|
| def test_extracts_multiple_numbers(self):
|
| """Test extracts multiple numbers."""
|
| text = "Account 123456 and phone 9876543210"
|
| result = extract_numbers(text)
|
| assert "123456" in result
|
| assert "9876543210" in result
|
|
|
| def test_handles_devanagari_digits(self):
|
| """Test handles Devanagari digits."""
|
| text = "Amount ५०००"
|
| result = extract_numbers(text)
|
| assert "5000" in result
|
|
|
| def test_no_numbers(self):
|
| """Test returns empty list when no numbers."""
|
| text = "No numbers here"
|
| result = extract_numbers(text)
|
| assert result == []
|
|
|
| def test_mixed_devanagari_and_latin(self):
|
| """Test mixed digit systems."""
|
| text = "Phone ९८76543२१० account 123"
|
| result = extract_numbers(text)
|
| assert "9876543210" in result
|
| assert "123" in result
|
|
|
|
|
| class TestMaskSensitiveData:
|
| """Tests for mask_sensitive_data function."""
|
|
|
| def test_masks_upi_id(self):
|
| """Test UPI ID is masked."""
|
| text = "Send to scammer@paytm"
|
| result = mask_sensitive_data(text)
|
| assert "scammer@paytm" not in result
|
| assert "[UPI_MASKED]" in result
|
|
|
| def test_masks_bank_account(self):
|
| """Test bank account number is masked."""
|
| text = "Account: 123456789012345"
|
| result = mask_sensitive_data(text)
|
| assert "123456789012345" not in result
|
| assert "[ACCOUNT_MASKED]" in result
|
|
|
| def test_masks_phone_number(self):
|
| """Test phone number is masked."""
|
| text = "Call 9876543210"
|
| result = mask_sensitive_data(text)
|
| assert "9876543210" not in result
|
|
|
| assert "[PHONE_MASKED]" in result or "[ACCOUNT_MASKED]" in result
|
|
|
| def test_masks_phone_with_plus91(self):
|
| """Test phone with +91 prefix is masked."""
|
| text = "Call +91 9876543210"
|
| result = mask_sensitive_data(text)
|
| assert "9876543210" not in result
|
|
|
| assert "[PHONE_MASKED]" in result or "[ACCOUNT_MASKED]" in result
|
|
|
| def test_preserves_non_sensitive_text(self):
|
| """Test non-sensitive text is preserved."""
|
| text = "Hello, how are you?"
|
| result = mask_sensitive_data(text)
|
| assert result == text
|
|
|
| def test_masks_multiple_sensitive_items(self):
|
| """Test masks multiple sensitive items in one text."""
|
| text = "Send to fraud@ybl, call 9876543210, account 123456789012"
|
| result = mask_sensitive_data(text)
|
|
|
| assert "fraud@ybl" not in result
|
| assert "9876543210" not in result
|
| assert "123456789012" not in result
|
|
|
|
|
| class TestPreprocessingEdgeCases:
|
| """Edge case tests for preprocessing functions."""
|
|
|
| def test_clean_text_with_emojis(self):
|
| """Test clean_text preserves emojis."""
|
| text = "Hello 😀 world 🎉"
|
| result = clean_text(text)
|
| assert "😀" in result
|
| assert "🎉" in result
|
|
|
| def test_normalize_very_long_text(self):
|
| """Test normalize handles very long text."""
|
| text = "word " * 10000
|
| result = normalize_text(text)
|
| assert len(result) > 0
|
|
|
| def test_devanagari_mixed_with_special_chars(self):
|
| """Test Devanagari digits with special characters."""
|
| text = "Amount: ₹५,०००/-"
|
| result = convert_devanagari_digits(text)
|
| assert "5" in result
|
| assert "0" in result
|
|
|
| def test_url_with_hindi_text(self):
|
| """Test URL removal with surrounding Hindi text."""
|
| text = "यहाँ क्लिक करें http://fake.com जीतने के लिए"
|
| result = remove_urls(text)
|
| assert "http://fake.com" not in result
|
| assert "यहाँ क्लिक करें" in result
|
|
|
