Upload 25 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,553 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .stTable {
+            margin-left: auto;
+            margin-right: auto;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    documentAssembler = DocumentAssembler()\
+            .setInputCol("text")\
+            .setOutputCol("document")
+
+    sentence_detector = SentenceDetector() \
+                .setInputCols(["document"]) \
+                .setOutputCol("sentence")
+
+    languageDetector = LanguageDetectorDL.pretrained(model)\
+                .setInputCols("sentence")\
+                .setOutputCol("language")\
+                .setThreshold(0.5)\
+                .setCoalesceSentences(True)
+
+    nlpPipeline = Pipeline(
+        stages=[ 
+            documentAssembler, 
+            sentence_detector,
+            languageDetector])
+
+    return nlpPipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)[0]
+
+    return results
+    
+# Set up the page layout
+st.markdown('<div class="main-title">State-Of-The-Art Language Detection With Spark NLP</div>', unsafe_allow_html=True)
+st.subheader('Support for 375 different languages')
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["ld_wiki_tatoeba_cnn_375"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+with st.expander("View Supported Languges"):
+    st.write("Abkhaz, Iraqi Arabic, Adyghe, Afrikaans, Gulf Arabic, Afrihili, Assyrian Neo-Aramaic, Ainu, Aklanon, Gheg Albanian, Amharic, Aragonese, Old English, Uab Meto, North Levantine Arabic, Arabic, Algerian Arabic, Moroccan Arabic, Egyptian Arabic, Assamese, Asturian, Kotava, Awadhi, Aymara, Azerbaijani, Bashkir, Baluchi, Balinese, Bavarian, Central Bikol, Belarusian, Berber, Bulgarian, Bhojpuri, Bislama, Banjar, Bambara, Bengali, Tibetan, Breton, Bodo, Bosnian, Buryat, Baybayanon, Brithenig, Catalan, Cayuga, Chavacano, Chechen, Cebuano, Chamorro, Chagatai, Chinook Jargon, Choctaw, Cherokee, Jin Chinese, Chukchi, Central Mnong, Corsican, Chinese Pidgin English, Crimean Tatar, Seychellois Creole, Czech, Kashubian, Chuvash, Welsh, CycL, Cuyonon, Danish, German, Dungan, Drents, Lower Sorbian, Central Dusun, Dhivehi, Dutton World Speedwords, Ewe, Emilian, Greek, Erromintxela, English, Middle English, Esperanto, Spanish, Estonian, Basque, Evenki, Extremaduran, Persian, Finnish, Fijian, Kven Finnish, Faroese, French, Middle French, Old French, North Frisian, Pulaar, Friulian, Nigerian Fulfulde, Frisian, Irish, Ga, Gagauz, Gan Chinese, Garhwali, Guadeloupean Creole French, Scottish Gaelic, Gilbertese, Galician, Guarani, Konkani (Goan), Gronings, Gothic, Ancient Greek, Swiss German, Gujarati, Manx, Hausa, Hakka Chinese, Hawaiian, Ancient Hebrew, Hebrew, Hindi, Fiji Hindi, Hiligaynon, Hmong Njua (Green), Ho, Croatian, Hunsrik, Upper Sorbian, Xiang Chinese, Haitian Creole, Hungarian, Armenian, Interlingua, Iban, Indonesian, Interlingue, Igbo, Nuosu, Inuktitut, Ilocano, Ido, Icelandic, Italian, Ingrian, Japanese, Jamaican Patois, Lojban, Juhuri (Judeo-Tat), Jewish Palestinian Aramaic, Javanese, Georgian, Karakalpak, Kabyle, Kamba, Kekchi (Q'eqchi'), Khasi, Khakas, Kazakh, Greenlandic, Khmer, Kannada, Korean, Komi-Permyak, Komi-Zyrian, Karachay-Balkar, Karelian, Kashmiri, Kölsch, Kurdish, Kumyk, Cornish, Keningau Murut, Kyrgyz, Coastal Kadazan, Latin, Southern Subanen, Ladino, Luxembourgish, Láadan, Lingua Franca Nova, Luganda, Ligurian, Livonian, Lakota, Ladin, Lombard, Lingala, Lao, Louisiana Creole, Lithuanian, Latgalian, Latvian, Latvian, Literary Chinese, Laz, Madurese, Maithili, North Moluccan Malay, Moksha, Morisyen, Malagasy, Mambae, Marshallese, Meadow Mari, Maori, Mi'kmaq, Minangkabau, Macedonian, Malayalam, Mongolian, Manchu, Mon, Mohawk, Marathi, Hill Mari, Malay, Maltese, Tagal Murut, Mirandese, Hmong Daw (White), Burmese, Erzya, Nauruan, Nahuatl, Norwegian Bokmål, Central Huasteca Nahuatl, Low German (Low Saxon), Nepali, Newari, Ngeq, Guerrero Nahuatl, Niuean, Dutch, Orizaba Nahuatl, Norwegian Nynorsk, Norwegian, Nogai, Old Norse, Novial, Nepali, Naga (Tangshang), Navajo, Chinyanja, Nyungar, Old Aramaic, Occitan, Ojibwe, Odia (Oriya), Old East Slavic, Ossetian, Old Spanish, Old Saxon, Ottoman Turkish, Old Turkish, Punjabi (Eastern), Pangasinan, Kapampangan, Papiamento, Palauan, Picard, Pennsylvania German, Palatine German, Phoenician, Pali, Polish, Piedmontese, Punjabi (Western), Pipil, Old Prussian, Pashto, Portuguese, Quechua, K'iche', Quenya, Rapa Nui, Rendille, Tarifit, Romansh, Kirundi, Romanian, Romani, Russian, Rusyn, Kinyarwanda, Okinawan, Sanskrit, Yakut, Sardinian, Sicilian, Scots, Sindhi, Northern Sami, Sango, Samogitian, Shuswap, Tachawit, Sinhala, Sindarin, Slovak, Slovenian, Samoan, Southern Sami, Shona, Somali, Albanian, Serbian, Swazi, Southern Sotho, Saterland Frisian, Sundanese, Sumerian, Swedish, Swahili, Swabian, Swahili, Syriac, Tamil, Telugu, Tetun, Tajik, Thai, Tahaggart Tamahaq, Tigrinya, Tigre, Turkmen, Tokelauan, Tagalog, Klingon, Talysh, Jewish Babylonian Aramaic, Temuan, Setswana, Tongan, Tonga (Zambezi), Toki Pona, Tok Pisin, Old Tupi, Turkish, Tsonga, Tatar, Isan, Tuvaluan, Tahitian, Tuvinian, Talossan, Udmurt, Uyghur, Ukrainian, Umbundu, Urdu, Urhobo, Uzbek, Venetian, Veps, Vietnamese, Volapük, Võro, Walloon, Waray, Wolof, Shanghainese, Kalmyk, Xhosa, Mingrelian, Yiddish, Yoruba, Cantonese, Chinese, Malay (Vernacular), Malay, Zulu, and Zaza.")
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/Language_Detector.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+folder_path = f"inputs/{model}"
+examples = [
+    lines[1].strip() 
+    for filename in os.listdir(folder_path) 
+    if filename.endswith('.txt') 
+    for lines in [open(os.path.join(folder_path, filename), 'r', encoding='utf-8').readlines()] 
+    if len(lines) >= 2
+]
+
+selected_text = st.selectbox("Select a sample text", examples)
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+elif selected_text:
+    selected_text = selected_text
+
+st.subheader('Selected Text')
+st.markdown("""<div class="section">{selected_text}</div>""", unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, selected_text)
+
+# Display output
+language_map = {
+    'ab': "Abkhaz",
+    'ace': "Achinese",
+    'acm': "Iraqi Arabic",
+    'ady': "Adyghe",
+    'af': "Afrikaans",
+    'afb': "Gulf Arabic",
+    'afh': "Afrihili",
+    'aii': "Assyrian Neo-Aramaic",
+    'ain': "Ainu",
+    'akl': "Aklanon",
+    'aln': "Gheg Albanian",
+    'als': "Tosk Albanian",
+    'am': "Amharic",
+    'an': "Aragonese",
+    'ang': "Old English",
+    'aoz': "Uab Meto",
+    'apc': "North Levantine Arabic",
+    'ar': "Arabic",
+    'arq': "Algerian Arabic",
+    'ary': "Moroccan Arabic",
+    'arz': "Egyptian Arabic",
+    'as': "Assamese",
+    'ast': "Asturian",
+    'av': "Avaric",
+    'avk': "Kotava",
+    'awa': "Awadhi",
+    'ay': "Aymara",
+    'az': "Azerbaijani",
+    'azb': "South Azerbaijani",
+    'ba': "Bashkir",
+    'bal': "Baluchi",
+    'ban': "Balinese",
+    'bar': "Bavarian",
+    'bat-smg': "bat-smg",
+    'bcl': "Central Bikol",
+    'be': "Belarusian",
+    'ber': "Berber",
+    'bg': "Bulgarian",
+    'bh': "bh",
+    'bho': "Bhojpuri",
+    'bi': "Bislama",
+    'bjn': "Banjar",
+    'bm': "Bambara",
+    'bn': "Bengali",
+    'bo': "Tibetan",
+    'bpy': "Bishnupriya",
+    'br': "Breton",
+    'brx': "Bodo",
+    'bs': "Bosnian",
+    'bua': "Buryat",
+    'bvy': "Baybayanon",
+    'bxr': "Russia Buriat",
+    'bzt': "Brithenig",
+    'ca': "Catalan",
+    'cay': "Cayuga",
+    'cbk': "Chavacano",
+    'cbk-zam': "cbk-zam",
+    'cdo': "Min Dong Chinese",
+    'ce': "Chechen",
+    'ceb': "Cebuano",
+    'ch': "Chamorro",
+    'chg': "Chagatai",
+    'chn': "Chinook Jargon",
+    'cho': "Choctaw",
+    'chr': "Cherokee",
+    'cjy': "Jin Chinese",
+    'ckb': "Central Kurdish (Soranî)",
+    'ckt': "Chukchi",
+    'cmo': "Central Mnong",
+    'co': "Corsican",
+    'cpi': "Chinese Pidgin English",
+    'crh': "Crimean Tatar",
+    'crs': "Seychellois Creole",
+    'cs': "Czech",
+    'ces': "Czech",
+    'csb': "Kashubian",
+    'cv': "Chuvash",
+    'cy': "Welsh",
+    'cycl': "CycL",
+    'cyo': "Cuyonon",
+    'da': "Danish",
+    'de': "German",
+    'deu': "German",
+    'diq': "Dimli (individual language)",
+    'dng': "Dungan",
+    'drt': "Drents",
+    'dsb': "Lower Sorbian",
+    'dtp': "Central Dusun",
+    'dty': "dty",
+    'dv': "Dhivehi",
+    'dws': "Dutton World Speedwords",
+    'ee': "Ewe",
+    'egl': "Emilian",
+    'el': "Greek",
+    'ell': "Greek",
+    'eml': "eml",
+    'emx': "Erromintxela",
+    'en': "English",
+    'enm': "Middle English",
+    'eo': "Esperanto",
+    'es': "Spanish",
+    'et': "Estonian",
+    'eu': "Basque",
+    'evn': "Evenki",
+    'ext': "Extremaduran",
+    'fa': "Persian",
+    'fi': "Finnish",
+    'fiu-vro': "fiu-vro",
+    'fj': "Fijian",
+    'fkv': "Kven Finnish",
+    'fo': "Faroese",
+    'fr': "French",
+    'fra': "French",
+    'frm': "Middle French",
+    'fro': "Old French",
+    'frp': "Arpitan",
+    'frr': "North Frisian",
+    'fuc': "Pulaar",
+    'fur': "Friulian",
+    'fuv': "Nigerian Fulfulde",
+    'fy': "Frisian",
+    'ga': "Irish",
+    'gaa': "Ga",
+    'gag': "Gagauz",
+    'gan': "Gan Chinese",
+    'gbm': "Garhwali",
+    'gcf': "Guadeloupean Creole French",
+    'gd': "Scottish Gaelic",
+    'gil': "Gilbertese",
+    'gl': "Galician",
+    'glk': "Gilaki",
+    'gn': "Guarani",
+    'gom': "Konkani (Goan)",
+    'gos': "Gronings",
+    'got': "Gothic",
+    'grc': "Ancient Greek",
+    'gsw': "Swiss German",
+    'gu': "Gujarati",
+    'gv': "Manx",
+    'ha': "Hausa",
+    'hak': "Hakka Chinese",
+    'haw': "Hawaiian",
+    'hbo': "Ancient Hebrew",
+    'he': "Hebrew",
+    'hi': "Hindi",
+    'hif': "Fiji Hindi",
+    'hil': "Hiligaynon",
+    'hnj': "Hmong Njua (Green)",
+    'hoc': "Ho",
+    'hr': "Croatian",
+    'hrx': "Hunsrik",
+    'hsb': "Upper Sorbian",
+    'hsn': "Xiang Chinese",
+    'ht': "Haitian Creole",
+    'hu': "Hungarian",
+    'hy': "Armenian",
+    'ia': "Interlingua",
+    'iba': "Iban",
+    'id': "Indonesian",
+    'ie': "Interlingue",
+    'ig': "Igbo",
+    'ii': "Nuosu",
+    'ike': "Inuktitut",
+    'ilo': "Ilocano",
+    'io': "Ido",
+    'is': "Icelandic",
+    'it': "Italian",
+    'izh': "Ingrian",
+    'ja': "Japanese",
+    'jam': "Jamaican Patois",
+    'jbo': "Lojban",
+    'jdt': "Juhuri (Judeo-Tat)",
+    'jpa': "Jewish Palestinian Aramaic",
+    'jv': "Javanese",
+    'ka': "Georgian",
+    'kaa': "Karakalpak",
+    'kab': "Kabyle",
+    'kam': "Kamba",
+    'kbd': "Kabardian",
+    'kek': "Kekchi (Q'eqchi')",
+    'kha': "Khasi",
+    'kjh': "Khakas",
+    'kk': "Kazakh",
+    'kl': "Greenlandic",
+    'km': "Khmer",
+    'kn': "Kannada",
+    'ko': "Korean",
+    'koi': "Komi-Permyak",
+    'kpv': "Komi-Zyrian",
+    'krc': "Karachay-Balkar",
+    'krl': "Karelian",
+    'ks': "Kashmiri",
+    'ksh': "Kölsch",
+    'ku': "Kurdish",
+    'kum': "Kumyk",
+    'kv': "Komi",
+    'kw': "Cornish",
+    'kxi': "Keningau Murut",
+    'ky': "Kyrgyz",
+    'kzj': "Coastal Kadazan",
+    'la': "Latin",
+    'laa': "Southern Subanen",
+    'lad': "Ladino",
+    'lb': "Luxembourgish",
+    'ldn': "Láadan",
+    'lez': "Lezghian",
+    'lfn': "Lingua Franca Nova",
+    'lg': "Luganda",
+    'li': "Limburgan",
+    'lij': "Ligurian",
+    'liv': "Livonian",
+    'lkt': "Lakota",
+    'lld': "Ladin",
+    'lmo': "Lombard",
+    'ln': "Lingala",
+    'lo': "Lao",
+    'lou': "Louisiana Creole",
+    'lrc': "Northern Luri",
+    'lt': "Lithuanian",
+    'ltg': "Latgalian",
+    'lv': "Latvian",
+    'lvs': "Latvian",
+    'lzh': "Literary Chinese",
+    'lzz': "Laz",
+    'mad': "Madurese",
+    'mai': "Maithili",
+    'map-bms': "map-bms",
+    'max': "North Moluccan Malay",
+    'mdf': "Moksha",
+    'mfe': "Morisyen",
+    'mg': "Malagasy",
+    'mgm': "Mambae",
+    'mh': "Marshallese",
+    'mhr': "Meadow Mari",
+    'mi': "Maori",
+    'mic': "Mi'kmaq",
+    'min': "Minangkabau",
+    'mk': "Macedonian",
+    'ml': "Malayalam",
+    'mn': "Mongolian",
+    'mnc': "Manchu",
+    'mnw': "Mon",
+    'moh': "Mohawk",
+    'mr': "Marathi",
+    'mrj': "Hill Mari",
+    'ms': "Malay",
+    'mt': "Maltese",
+    'mvv': "Tagal Murut",
+    'mwl': "Mirandese",
+    'mww': "Hmong Daw (White)",
+    'my': "Burmese",
+    'myv': "Erzya",
+    'mzn': "Mazanderani",
+    'na': "Nauruan",
+    'nah': "Nahuatl",
+    'nap': "Neapolitan",
+    'nb': "Norwegian Bokmål",
+    'nch': "Central Huasteca Nahuatl",
+    'nds': "Low German (Low Saxon)",
+    'nds-nl': "nds-nl",
+    'ne': "Nepali",
+    'new': "Newari",
+    'ngt': "Ngeq",
+    'ngu': "Guerrero Nahuatl",
+    'niu': "Niuean",
+    'nl': "Dutch",
+    'nlv': "Orizaba Nahuatl",
+    'nn': "Norwegian Nynorsk",
+    'no': "Norwegian",
+    'nog': "Nogai",
+    'non': "Old Norse",
+    'nov': "Novial",
+    'npi': "Nepali",
+    'nrm': "Narom",
+    'nso': "Pedi",
+    'nst': "Naga (Tangshang)",
+    'nv': "Navajo",
+    'ny': "Chinyanja",
+    'nys': "Nyungar",
+    'oar': "Old Aramaic",
+    'oc': "Occitan",
+    'oj': "Ojibwe",
+    'olo': "Livvi",
+    'om': "Oromo",
+    'or': "Odia (Oriya)",
+    'orv': "Old East Slavic",
+    'os': "Ossetian",
+    'osp': "Old Spanish",
+    'osx': "Old Saxon",
+    'ota': "Ottoman Turkish",
+    'otk': "Old Turkish",
+    'pa': "Punjabi (Eastern)",
+    'pag': "Pangasinan",
+    'pam': "Kapampangan",
+    'pap': "Papiamento",
+    'pau': "Palauan",
+    'pcd': "Picard",
+    'pdc': "Pennsylvania German",
+    'pfl': "Palatine German",
+    'phn': "Phoenician",
+    'pi': "Pali",
+    'pl': "Polish",
+    'pms': "Piedmontese",
+    'pnb': "Punjabi (Western)",
+    'ppl': "Pipil",
+    'prg': "Old Prussian",
+    'ps': "Pashto",
+    'pt': "Portuguese",
+    'qu': "Quechua",
+    'quc': "K'iche'",
+    'qya': "Quenya",
+    'rap': "Rapa Nui",
+    'rel': "Rendille",
+    'rif': "Tarifit",
+    'rm': "Romansh",
+    'rn': "Kirundi",
+    'ro': "Romanian",
+    'ron': "Romanian",
+    'roa-rup': "roa-rup",
+    'roa-tara': "roa-tara",
+    'rom': "Romani",
+    'ru': "Russian",
+    'rue': "Rusyn",
+    'rw': "Kinyarwanda",
+    'ryu': "Okinawan",
+    'sa': "Sanskrit",
+    'sah': "Yakut",
+    'sc': "Sardinian",
+    'scn': "Sicilian",
+    'sco': "Scots",
+    'sd': "Sindhi",
+    'se': "Northern Sami",
+    'sg': "Sango",
+    'sgs': "Samogitian",
+    'sh': "Serbo-Croatian",
+    'shs': "Shuswap",
+    'shy': "Tachawit",
+    'si': "Sinhala",
+    'sjn': "Sindarin",
+    'sk': "Slovak",
+    'slk': "Slovak",
+    'sl': "Slovenian",
+    'sm': "Samoan",
+    'sma': "Southern Sami",
+    'sn': "Shona",
+    'so': "Somali",
+    'sq': "Albanian",
+    'sr': "Serbian",
+    'srn': "Sranan Tongo",
+    'ss': "Swazi",
+    'st': "Southern Sotho",
+    'stq': "Saterland Frisian",
+    'su': "Sundanese",
+    'sux': "Sumerian",
+    'sv': "Swedish",
+    'sw': "Swahili",
+    'swg': "Swabian",
+    'swh': "Swahili",
+    'syc': "Syriac",
+    'szl': "Silesian",
+    'ta': "Tamil",
+    'tcy': "Tulu",
+    'te': "Telugu",
+    'tet': "Tetun",
+    'tg': "Tajik",
+    'th': "Thai",
+    'thv': "Tahaggart Tamahaq",
+    'ti': "Tigrinya",
+    'tig': "Tigre",
+    'tk': "Turkmen",
+    'tkl': "Tokelauan",
+    'tl': "Tagalog",
+    'tlh': "Klingon",
+    'tly': "Talysh",
+    'tmr': "Jewish Babylonian Aramaic",
+    'tmw': "Temuan",
+    'tn': "Setswana",
+    'to': "Tongan",
+    'toi': "Tonga (Zambezi)",
+    'toki': "Toki Pona",
+    'tpi': "Tok Pisin",
+    'tpw': "Old Tupi",
+    'tr': "Turkish",
+    'ts': "Tsonga",
+    'tt': "Tatar",
+    'tts': "Isan",
+    'tvl': "Tuvaluan",
+    'ty': "Tahitian",
+    'tyv': "Tuvinian",
+    'tzl': "Talossan",
+    'udm': "Udmurt",
+    'ug': "Uyghur",
+    'uk': "Ukrainian",
+    'umb': "Umbundu",
+    'ur': "Urdu",
+    'urh': "Urhobo",
+    'uz': "Uzbek",
+    'vec': "Venetian",
+    'vep': "Veps",
+    'vi': "Vietnamese",
+    'vls': "Vlaams",
+    'vo': "Volapük",
+    'vro': "Võro",
+    'wa': "Walloon",
+    'war': "Waray",
+    'wo': "Wolof",
+    'wuu': "Shanghainese",
+    'xal': "Kalmyk",
+    'xh': "Xhosa",
+    'xmf': "Mingrelian",
+    'yi': "Yiddish",
+    'yo': "Yoruba",
+    'yue': "Cantonese",
+    'zea': "Zeeuws",
+    'zh': "Chinese",
+    'zh-classical': "zh-classical",
+    'zh-min-nan': "zh-min-nan",
+    'zh-yue': "zh-yue",
+    'zlm': "Malay (Vernacular)",
+    'zsm': "Malay",
+    'zu': "Zulu",
+    'zza': "Zaza"
+  }
+
+language = language_map[output['language'][0].result]
+confidence = round(float(output['language'][0].metadata[language])*100, 2)
+
+st.markdown(f"This text is in **{language} ({output['language'][0].result})** language.")
+st.markdown(f"Classification Confidence: **{confidence}%**")

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

inputs/ld_wiki_tatoeba_cnn_375/Example1.txt

@@ -0,0 +1,2 @@
+Mona Lisa is a 16th century oil painting created by Leonardo. It is held at the Louvre... 
+Mona Lisa is a 16th century oil painting created by Leonardo. It is held at the Louvre in Paris.

inputs/ld_wiki_tatoeba_cnn_375/Example10.txt

@@ -0,0 +1,2 @@
+La Joconde est une peinture à l'huile du XVIe siècle créée par Léonard. Il se tient au Louvr... 
+La Joconde est une peinture à l'huile du XVIe siècle créée par Léonard. Il se tient au Louvre à Paris.

inputs/ld_wiki_tatoeba_cnn_375/Example11.txt

@@ -0,0 +1,2 @@
+Quando Sebastian Thrun ha iniziato a lavorare su auto a guida autonoma presso Google nel 2007, poche... 
+Quando Sebastian Thrun ha iniziato a lavorare su auto a guida autonoma presso Google nel 2007, poche persone al di fuori dell'azienda lo hanno preso sul serio.

inputs/ld_wiki_tatoeba_cnn_375/Example12.txt

@@ -0,0 +1,2 @@
+Mona Lisa je olejomalba ze 16. století, kterou vytvořil Leonardo. Koná se v Louvru... 
+Mona Lisa je olejomalba ze 16. století, kterou vytvořil Leonardo. Koná se v Louvru v Paříži.

inputs/ld_wiki_tatoeba_cnn_375/Example13.txt

@@ -0,0 +1,2 @@
+Джефри Еверест Хинтън е британски канадски когнитивен психолог и компютърен учен, най-известен с раб... 
+Джефри Еверест Хинтън е британски канадски когнитивен психолог и компютърен учен, най-известен с работата си върху изкуствени невронни мрежи. От 2013 г. той прекарва времето си в работа за Google и университета в Торонто. През 2017 г. е съосновател и става главен научен съветник на Vector Institute of Toronto.

inputs/ld_wiki_tatoeba_cnn_375/Example14.txt

@@ -0,0 +1,2 @@
+Titanic ist ein 1997 in den USA veröffentlichter epischer Roman und ein katastrophaler Film, der von... 
+Titanic ist ein 1997 in den USA veröffentlichter epischer Roman und ein katastrophaler Film, der von James Cameron inszeniert, geschrieben, co-produziert und mitherausgegeben wurde. Es deckt sowohl historische als auch fiktive Aspekte ab und basiert auf Berichten über den Untergang der RMS Titanic und der Stars Leonard DiCaprio und Kate Winslet als Mitglieder verschiedener sozialer Klassen, die sich während der Schiffsreise während ihrer unglücklichen ersten Reise verlieben.

inputs/ld_wiki_tatoeba_cnn_375/Example15.txt

@@ -0,0 +1,2 @@
+Geoffrey Everest Hinton es un psicólogo cognitivo y científico informático británico canadiense, mej... 
+Geoffrey Everest Hinton es un psicólogo cognitivo y científico informático británico canadiense, mejor conocido por su trabajo en redes neuronales artificiales. Desde 2013 ha pasado su tiempo trabajando para Google y la Universidad de Toronto. En 2017 fue cofundador y se convirtió en Asesor Científico Jefe del Instituto Vector de Toronto.

inputs/ld_wiki_tatoeba_cnn_375/Example16.txt

@@ -0,0 +1,2 @@
+Το Titanic είναι ένα αμερικανικό επικό μυθιστόρημα του 1997 και μια καταστροφική ταινία σε σκηνοθεσί... 
+Το Titanic είναι ένα αμερικανικό επικό μυθιστόρημα του 1997 και μια καταστροφική ταινία σε σκηνοθεσία, συγγραφή, συμπαραγωγή και συν-επεξεργασία από τον James Cameron. Καλύπτει τόσο ιστορικές όσο και φανταστικές πτυχές και βασίζεται σε αναφορές για τη βύθιση του Τιτανικού RMS και πρωταγωνιστούν οι Leonard DiCaprio και Kate Winslet ως μέλη διαφόρων κοινωνικών τάξεων που ερωτεύονται κατά τη διάρκεια του ταξιδιού του πλοίου κατά τη διάρκεια του πρώτου τους ατυχούς ταξιδιού.

inputs/ld_wiki_tatoeba_cnn_375/Example17.txt

@@ -0,0 +1,2 @@
+Mona Lisa on 1500-luvun öljymaalaus, jonka on luonut Leonardo. Se pidetään Pariisin ... 
+Mona Lisa on 1500-luvun öljymaalaus, jonka on luonut Leonardo. Se pidetään Pariisin Louvressa.

inputs/ld_wiki_tatoeba_cnn_375/Example18.txt

@@ -0,0 +1,2 @@
+A természetes nyelvfeldolgozás története általában az 1950-es években kezdődött, bár a korábbi idősz... 
+A természetes nyelvfeldolgozás története általában az 1950-es években kezdődött, bár a korábbi időszakokból származó munkák is megtalálhatók. 1950-ben Alan Turing közzétett egy cikket, melynek címe: „Számítástechnika és intelligenciagépek”, és amely intelligenciakritériumként javasolta a Turing-tesztet.

inputs/ld_wiki_tatoeba_cnn_375/Example19.txt

@@ -0,0 +1,2 @@
+Povijest obrade prirodnog jezika općenito je počela 1950-ih, iako se mogu naći djela iz ranijih razd... 
+Povijest obrade prirodnog jezika općenito je počela 1950-ih, iako se mogu naći djela iz ranijih razdoblja. 1950. Alan Turing objavio je članak pod naslovom "Računalna i inteligencijska mašinerija" u kojem se kao kriterij inteligencije predlaže ono što se danas naziva Turingov test.

inputs/ld_wiki_tatoeba_cnn_375/Example2.txt

@@ -0,0 +1,2 @@
+Poza tym, że jest królem północy, John Snow jest angielskim lekarzem i liderem w dziedzinie anestezj... 
+Poza tym, że jest królem północy, John Snow jest angielskim lekarzem i liderem w dziedzinie anestezjologii i higieny medycznej. Uważany jest za pierwszego, który wykorzystał dane do leczenia epidemii cholery w 1834 r.

inputs/ld_wiki_tatoeba_cnn_375/Example20.txt

@@ -0,0 +1,2 @@
+Titanic er en amerikansk episk roman fra 1997 og katastrofal film regissert, skrevet, co-produsert o... 
+Titanic er en amerikansk episk roman fra 1997 og katastrofal film regissert, skrevet, co-produsert og co-redigert av James Cameron. Det dekker både historiske og fiksjoniserte aspekter og er basert på rapporter om synkingen av RMS Titanic og stjernene Leonard DiCaprio og Kate Winslet som medlemmer av forskjellige sosiale klasser som forelsker seg under skipets seilas under deres uheldige første seilas.

inputs/ld_wiki_tatoeba_cnn_375/Example3.txt

@@ -0,0 +1,2 @@
+Geoffrey Everest Hinton é um psicólogo cognitivo britânico canadense e cientista da computação, mais... 
+Geoffrey Everest Hinton é um psicólogo cognitivo britânico canadense e cientista da computação, mais conhecido por seu trabalho em redes neurais artificiais. Desde 2013, ele trabalha para o Google e a Universidade de Toronto. Em 2017, foi co-fundador e tornou-se Conselheiro Científico Chefe do Vector Institute of Toronto.

inputs/ld_wiki_tatoeba_cnn_375/Example4.txt

@@ -0,0 +1,2 @@
+Когда в 2007 году Себастьян Трун начал работать над машинами для самостоятельного вождения в Google,... 
+Когда в 2007 году Себастьян Трун начал работать над машинами для самостоятельного вождения в Google, мало кто за пределами компании воспринимал его всерьез.

inputs/ld_wiki_tatoeba_cnn_375/Example5.txt

@@ -0,0 +1,2 @@
+Mona Lisa este o pictură în ulei din secolul al XVI-lea creată de Leonardo. Se ține la Louvre ... 
+Mona Lisa este o pictură în ulei din secolul al XVI-lea creată de Leonardo. Se ține la Louvre din Paris.

inputs/ld_wiki_tatoeba_cnn_375/Example6.txt

@@ -0,0 +1,2 @@
+John Snow je okrem anglického kráľa anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygie... 
+John Snow je okrem anglického kráľa anglickým lekárom a lídrom vo vývoji anestézie a lekárskej hygieny. Je považovaný za prvého, ktorý používa údaje na liečenie prepuknutia cholery v roku 1834.

inputs/ld_wiki_tatoeba_cnn_375/Example7.txt

@@ -0,0 +1,2 @@
+Mona Lisa, Leonardo tarafından yaratılan 16. yüzyıldan kalma bir yağlı boyadır. Paris'teki Louvre'da... 
+Mona Lisa, Leonardo tarafından yaratılan 16. yüzyıldan kalma bir yağlı boyadır. Paris'teki Louvre'da düzenleniyor.

inputs/ld_wiki_tatoeba_cnn_375/Example8.txt

@@ -0,0 +1,2 @@
+Mona Lisa är en oljemålning från 1500-talet skapad av Leonardo. Det hålls vid Louvr... 
+Mona Lisa är en oljemålning från 1500-talet skapad av Leonardo. Det hålls vid Louvre i Paris.

inputs/ld_wiki_tatoeba_cnn_375/Example9.txt

@@ -0,0 +1,2 @@
+Facebook - це послуга соціальних мереж, запущена під назвою TheFacebook 4 лютого 2004 року. Його зас... 
+Facebook - це послуга соціальних мереж, запущена під назвою TheFacebook 4 лютого 2004 року. Його заснував Марк Цукерберг разом зі своїми одноквартами та колегами Гарвардського університету Едуардо Саверином, Ендрю Макколлумом, Дастіном Московіцем та Крісом Х'юзом.

pages/Workflow & Model Overview.py

@@ -0,0 +1,278 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Title
+st.markdown('<div class="main-title">Automatic Language Detection Using Spark NLP in Python</div>', unsafe_allow_html=True)
+
+# Introduction
+st.markdown("""
+<div class="section">
+    <p>Language detection is a critical component of Natural Language Processing (NLP), which involves automatically identifying the language of a given piece of text. This functionality is essential in various multilingual applications where the language of input text might not be known in advance. Accurate language detection can enhance the performance of downstream NLP tasks such as machine translation, sentiment analysis, and information retrieval.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# What is Language Detection
+st.markdown('<div class="sub-title">What is Language Detection?</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Language detection models analyze text to determine its language by examining features such as:</p>
+    <ul>
+        <li><b>Character Set</b>: Identifying language-specific characters and symbols.</li>
+        <li><b>Word Frequency</b>: Recognizing common words and their usage patterns in different languages.</li>
+        <li><b>N-grams</b>: Analyzing sequences of n words to detect language-specific phrases and structures.</li>
+    </ul>
+    <p>Models are typically trained on extensive datasets (e.g., Wikipedia, Tatoeba) using statistical and deep learning methods to recognize these patterns. Once trained, these models can predict the language of new text by comparing its features with those learned during training.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Importance and Use Cases
+st.markdown('<div class="sub-title">Importance and Use Cases</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Accurate language detection is pivotal for many applications, including:</p>
+    <ul>
+        <li><b>Machine Translation</b>: Automatically translating text into various languages.</li>
+        <li><b>Sentiment Analysis</b>: Analyzing sentiments in multilingual datasets.</li>
+        <li><b>Information Retrieval</b>: Enhancing search results by filtering content based on language.</li>
+        <li><b>Spam Filtering</b>: Identifying spam content in multiple languages.</li>
+        <li><b>Social Media Analysis</b>: Processing and categorizing user-generated content in different languages.</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Spark NLP's LanguageDetectorDL
+st.markdown('<div class="sub-title">Spark NLP\'s LanguageDetectorDL</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The <code>LanguageDetectorDL</code> annotator from Spark NLP is designed for high accuracy in language detection. It utilizes pretrained deep learning models to identify languages with precision. This annotator can effectively handle documents containing mixed languages by analyzing sentence segments and selecting the most probable language.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP and extract keywords in Python, simply use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+st.markdown('<p>For other installation options and environments, refer to the <a href="https://nlp.johnsnowlabs.com/docs/en/install" class="link">official documentation</a>.</p>', unsafe_allow_html=True)
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# Using LanguageDetectorDL
+st.markdown('<div class="sub-title">Using LanguageDetectorDL</div>', unsafe_allow_html=True)
+st.code("""
+# Import necessary modules
+from sparknlp.base import DocumentAssembler, Pipeline
+from sparknlp.annotator import LanguageDetectorDL
+import pyspark.sql.functions as F
+
+# Step 1: Transform raw text into `document` annotation
+document_assembler = (
+    DocumentAssembler()
+    .setInputCol("text")
+    .setOutputCol("document")
+)
+
+# Step 2: Detect the language of the text
+language_detector = (
+    LanguageDetectorDL.pretrained()
+    .setInputCols("document")
+    .setOutputCol("language") 
+)
+
+# Create the NLP pipeline
+nlpPipeline = Pipeline(stages=[document_assembler, language_detector])
+
+# Sample texts in different languages
+data = spark.createDataFrame([
+    ["Spark NLP is an open-source text processing library for advanced natural language processing for the Python, Java and Scala programming languages."],
+    ["Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du langage naturel pour les langages de programmation Python, Java et Scala."],
+    ["Spark NLP ist eine Open-Source-Textverarbeitungsbibliothek für fortgeschrittene natürliche Sprachverarbeitung für die Programmiersprachen Python, Java und Scala."],
+    ["Spark NLP es una biblioteca de procesamiento de texto de código abierto para el procesamiento avanzado de lenguaje natural para los lenguajes de programación Python, Java y Scala."],
+    ["Spark NLP é uma biblioteca de processamento de texto de código aberto para processamento avançado de linguagem natural para as linguagens de programação Python, Java e Scala"]
+]).toDF("text")
+
+# Transform the data with the pipeline
+result = nlpPipeline.fit(data).transform(data)
+
+# Show the results
+result.select("text", "language.result").show(truncate=100)
+""", language='python')
+
+st.text("""
++----------------------------------------------------------------------------------------------------+------+
+|                                                                                                text|result|
++----------------------------------------------------------------------------------------------------+------+
+|Spark NLP is an open-source text processing library for advanced natural language processing for ...|  [en]|
+|Spark NLP est une bibliothèque de traitement de texte open source pour le traitement avancé du la...|  [fr]|
+|Spark NLP ist eine Open-Source-Textverarbeitungsbibliothek für fortgeschrittene natürliche Sprach...|  [de]|
+|Spark NLP es una biblioteca de procesamiento de texto de código abierto para el procesamiento ava...|  [es]|
+|Spark NLP é uma biblioteca de processamento de texto de código aberto para processamento avançado...|  [pt]|
++----------------------------------------------------------------------------------------------------+------+
+""")
+
+# One-Liner Alternative
+st.markdown('<div class="sub-title">One-Liner Alternative</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>John Snow Labs has introduced a unified library to simplify workflows across various products, including Spark NLP. Install the library with:</p>
+    <pre><code>pip install johnsnowlabs</code></pre>
+    <p>Use the following one-liner code for quick language detection:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+# Import the NLP module which contains Spark NLP and NLU libraries
+from johnsnowlabs import nlp
+
+# Sample text in Polish
+sample_text = "Spark NLP to biblioteka edytorów tekstu typu open source do zaawansowanego przetwarzania języka naturalnego w językach programowania Python, Java i Scala."
+
+# Detect language with one line of code
+result = nlp.load('xx.classify.wiki_95').predict(sample_text, output_level='sentence')
+""", language='python')
+
+st.markdown("""
+<table style="width:100%; border-collapse: collapse; margin-top: 20px;">
+  <thead>
+    <tr style="background-color: #4A90E2; color: white; text-align: left;">
+      <th style="padding: 12px;">Language</th>
+      <th style="padding: 12px;">Confidence</th>
+      <th style="padding: 12px;">Sentence</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr style="background-color: #f9f9f9;">
+      <td style="padding: 12px; border: 1px solid #ddd;">pl</td>
+      <td style="padding: 12px; border: 1px solid #ddd;">9.0</td>
+      <td style="padding: 12px; border: 1px solid #ddd;">Spark NLP to biblioteka edytorów tekstu typu open source do zaawansowanego przetwarzania języka naturalnego w językach programowania Python, Java i Scala.</td>
+    </tr>
+  </tbody>
+</table>
+""", unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p><b>Benefits of the One-Liner</b></p>
+    <p>This approach is convenient for quick implementations and testing. The one-liner model is based on default configurations, which may suffice for general use cases. However, for more specialized needs, customizing the pipeline or choosing specific models might be necessary.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Notes and Recommendations
+st.markdown('<div class="sub-title">Notes and Recommendations</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><b>Customizing Pipelines</b>: While the one-liner is efficient, building a custom pipeline with specific models and configurations allows for greater flexibility and optimization according to the application's requirements.</li>
+        <li><b>Handling Mixed Languages</b>: <code>LanguageDetectorDL</code> can effectively manage texts with multiple languages by analyzing sentence segments. Ensure your pipeline is configured to handle such cases appropriately.</li>
+        <li><b>Performance Considerations</b>: When working with large datasets, optimizing Spark configurations and resources is crucial for maintaining performance and avoiding bottlenecks.</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Benchmarking Section
+st.markdown('<div class="sub-title">Benchmarking</div>', unsafe_allow_html=True)
+st.write("")
+st.markdown('<p><a href="https://sparknlp.org/2020/12/05/ld_wiki_tatoeba_cnn_375_xx.html" class="link" target="_blank">ld_wiki_tatoeba_cnn_375</a> Model Evaluated on Europarl dataset which the model has never seen:</p>', unsafe_allow_html=True)
+st.text("""
++--------+-----+-------+------------------+     
+|src_lang|count|correct|         precision|       
++--------+-----+-------+------------------+  
+|      fr| 1000|   1000|               1.0|    
+|      de| 1000|    999|             0.999|       
+|      fi| 1000|    999|             0.999|              
+|      nl| 1000|    998|             0.998|          +-------+--------------------+ 
+|      el| 1000|    997|             0.997|          |summary|           precision|
+|      en| 1000|    995|             0.995|          +-------+--------------------+  
+|      es| 1000|    994|             0.994|          |  count|                  21| 
+|      it| 1000|    993|             0.993|          |   mean|  0.9758952066282511|   
+|      sv| 1000|    991|             0.991|          | stddev|0.029434744995013935|   
+|      da| 1000|    987|             0.987|          |    min|  0.8862144420131292|
+|      pl|  914|    901|0.9857768052516411|          |    max|                 1.0| 
+|      hu|  880|    866|0.9840909090909091|          +-------+--------------------+
+|      pt| 1000|    980|              0.98|                         
+|      et|  928|    907|0.9773706896551724| 
+|      ro|  784|    766|0.9770408163265306|
+|      lt| 1000|    976|             0.976|
+|      bg| 1000|    965|             0.965|
+|      cs| 1000|    945|             0.945|
+|      sk| 1000|    944|             0.944|
+|      lv|  916|    843|0.9203056768558951|
+|      sl|  914|    810|0.8862144420131292|
++--------+-----+-------+------------------+
+""")
+
+# Conclusion
+st.markdown("""
+<div class="section">
+    <h2>Conclusion</h2>
+    <p>Accurate language detection is a foundational step in many NLP workflows. Spark NLP’s <code>LanguageDetectorDL</code> annotator offers a robust solution for detecting languages in diverse text corpora. With its integration into Spark's powerful data processing framework, it enables efficient handling of large-scale multilingual datasets, providing accurate language identification for various applications.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">References and Additional Information</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a href="https://nlp.johnsnowlabs.com/docs/en/annotators#languagedetectordl" class="link" target="_blank">Documentation: LanguageDetectorDL</a></li>
+        <li><a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/language_detector_dl/index.html#sparknlp.annotator.language_detector_dl.LanguageDetectorDL" class="link" target="_blank">Python Docs: LanguageDetectorDL</a></li>
+        <li><a href="https://sparknlp.org/2020/12/05/ld_wiki_tatoeba_cnn_375_xx.html" class="link" target="_blank">ld_wiki_tatoeba_cnn_375</a></li>
+        <li><a href="https://www.johnsnowlabs.com/how-to-detect-languages-with-python-a-comprehensive-guide/" class="link" target="_blank">Reference Article</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark