Upload 58 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,190 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    documentAssembler = DocumentAssembler()\
+          .setInputCol("text")\
+          .setOutputCol("document")
+
+    sentenceDetector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "xx")\
+          .setInputCols(["document"])\
+          .setOutputCol("sentence")
+
+    tokenizer = Tokenizer()\
+          .setInputCols(["sentence"])\
+          .setOutputCol("token")
+
+    ner_converter = NerConverter()\
+          .setInputCols(["sentence", "token", "ner"])\
+          .setOutputCol("ner_chunk")
+
+
+    if model == 'xlm_roberta_large_token_classifier_masakhaner':
+      tokenClassifier = XlmRoBertaForTokenClassification.pretrained("xlm_roberta_large_token_classifier_masakhaner", "xx")\
+          .setInputCols(["sentence",'token'])\
+          .setOutputCol("ner")
+
+    else:
+      tokenClassifier = DistilBertForTokenClassification.pretrained("distilbert_base_token_classifier_masakhaner", "xx")\
+          .setInputCols(["sentence",'token'])\
+          .setOutputCol("ner")
+
+    nlpPipeline = Pipeline(stages=[documentAssembler, sentenceDetector, tokenizer, tokenClassifier, ner_converter])
+    return nlpPipeline
+
+def fit_data(pipeline, data):
+  empty_df = spark.createDataFrame([['']]).toDF('text')
+  pipeline_model = pipeline.fit(empty_df)
+  model = LightPipeline(pipeline_model)
+  result = model.fullAnnotate(data)
+  return result
+
+def annotate(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+# Set up the page layout
+st.markdown('<div class="main-title">Recognize entities in 10 African languages</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>This model carries out Name Entity Recognition on 10 African languages (Amharic, Hausa, Igbo, Kinyarwanda, Luganda, Nigerian, Pidgin, Swahilu, Wolof, and Yorùbá).</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["xlm_roberta_large_token_classifier_masakhaner", "distilbert_base_token_classifier_masakhaner"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+language = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["Amharic", "Hausa", "Igbo", "Kinyarwanda", "Luganda", "Nigerian", "Pidgin", "Swahilu", "Wolof", "Yorùbá"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+try:
+    labels_set = set()
+    for i in results['NER Chunk'].values:
+        labels_set.add(results["NER Label"][i])
+        labels_set = list(labels_set)
+
+    labels = st.sidebar.multiselect("Entity labels", options=labels_set, default=list(labels_set))
+        
+    NER_labs = ['PER', 'ORG', 'LOC', 'DATE']
+    NER_exp = ['People, including fictional.', 'Companies, agencies, institutions, etc.', 'Countries, cities, states.', 'Date, Year']
+
+    NER_dict = dict(zip(NER_labs, NER_exp))
+
+    show_exp = st.sidebar.checkbox("Explain NER Labels", value=True)
+    if show_exp:
+        t_ner_k = []
+        t_ner_v = []
+    for t_lab in labels_set:
+        if t_lab in NER_dict:
+            t_ner_k.append(t_lab)
+            t_ner_v.append(NER_dict[t_lab])
+    tdf = pd.DataFrame({"NER": t_ner_k, "Meaning": t_ner_v})
+    tdf.index=['']*len(t_ner_k)
+    st.sidebar.table(tdf)
+except:
+    pass
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/Ner_masakhaner.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+folder_path = f"inputs/{language}"
+examples = [
+    lines[1].strip() 
+    for filename in os.listdir(folder_path) 
+    if filename.endswith('.txt') 
+    for lines in [open(os.path.join(folder_path, filename), 'r', encoding='utf-8').readlines()] 
+    if len(lines) >= 2
+]
+
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+
+results = {
+    'Document': output[0]['document'][0].result,
+    'NER Chunk': [n.result for n in output[0]['ner_chunk']],
+    "NER Label": [n.metadata['entity'] for n in output[0]['ner_chunk']]
+}
+
+annotate(results)
+
+with st.expander("View DataFrame"):
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

inputs/Amharic/Example1.txt

@@ -0,0 +1,2 @@
+አህመድ ቫንዳ ከ3-10-2000 ጀምሮ በአዲ... 
+አህመድ ቫንዳ ከ3-10-2000 ጀምሮ በአዲስ አበባ ኖሯል።

inputs/Amharic/Example2.txt

@@ -0,0 +1,2 @@
+ሰማያዊ ፓርቲ ዛሬ በወቅታዊ የሀገሪቱ የፖለቲካ ጉዳዮች ላይ በመኢአድ ጽህፈት ቤት የሰጠው ጋዜጣዊ መግለ... 
+ሰማያዊ ፓርቲ ዛሬ በወቅታዊ የሀገሪቱ የፖለቲካ ጉዳዮች ላይ በመኢአድ ጽህፈት ቤት የሰጠው ጋዜጣዊ መግለጫ ይከተላል ፡፡

inputs/Amharic/Example3.txt

@@ -0,0 +1,2 @@
+የ ዓመቱ አዲሱ የዚምባብዌ ፕሬዚደንት ኤመርሰን ምናንጋግዋ በሁለቱ ቻዎቻቸው አንፃር በዕድሜ... 
+የ ዓመቱ አዲሱ የዚምባብዌ ፕሬዚደንት ኤመርሰን ምናንጋግዋ በሁለቱ ቻዎቻቸው አንፃር በዕድሜ ትንሹ ናቸው ።

inputs/Amharic/Example4.txt

@@ -0,0 +1,2 @@
+ዶይቸ ቬለ ያነጋገራቸው የመብት ተሟጋቿ ሊንዳ ማዜሪሬ በዕድሙ በተዳከሙ መሪዎች ነው የምንተዳደረው በማለት የርሳቸውን እና የአህጉሩን ወጣት ትውልድ ቅ... 
+ዶይቸ ቬለ ያነጋገራቸው የመብት ተሟጋቿ ሊንዳ ማዜሪሬ በዕድሙ በተዳከሙ መሪዎች ነው የምንተዳደረው በማለት የርሳቸውን እና የአህጉሩን ወጣት ትውልድ ቅሬታ ገልጸዋል ።

inputs/Amharic/Example5.txt

@@ -0,0 +1,2 @@
+ሪታ ፓንክኸርስት የኢትዮጵያ ባለውለታ አዲስ አበባ ላይ ትዳር የመሰረቱት የኢትዮጵያ ታሪክ ተመራማሪዎች በትዳር ከ ዓመታት በ... 
+ሪታ ፓንክኸርስት የኢትዮጵያ ባለውለታ አዲስ አበባ ላይ ትዳር የመሰረቱት የኢትዮጵያ ታሪክ ተመራማሪዎች በትዳር ከ ዓመታት በላይ ዘልቀዋል ።

inputs/Amharic/Example6.txt

@@ -0,0 +1,2 @@
+በሳልስቱ እስራኤል ዉስጥ በተደረገዉ አጠቃላይ ምርጫ አክራሪዉ የጠቅላይ ሚንስትር ቤንያሚን ኔትንያሁ ፓርቲ ... 
+በሳልስቱ እስራኤል ዉስጥ በተደረገዉ አጠቃላይ ምርጫ አክራሪዉ የጠቅላይ ሚንስትር ቤንያሚን ኔትንያሁ ፓርቲ ሊኩድ አሸነፈ ።

inputs/Hausa/Example1.txt

@@ -0,0 +1,2 @@
+A saurari cikakken rahoton wakilin Muryar Amurka Ibrahim ... 
+A saurari cikakken rahoton wakilin Muryar Amurka Ibrahim Abdul'aziz

inputs/Hausa/Example2.txt

@@ -0,0 +1,2 @@
+Najeriya : Kungiyar Ma'aikatan Jami'o'i Ta Shiga Yajin Aik... 
+Najeriya : Kungiyar Ma'aikatan Jami'o'i Ta Shiga Yajin Aikin Gargadi

inputs/Hausa/Example3.txt

@@ -0,0 +1,2 @@
+A ranar Juma’a mai zuwa ne wa’adin yajin aikin na gargadi zai kammala , kuma a hirar su da wakilin M... 
+A ranar Juma’a mai zuwa ne wa’adin yajin aikin na gargadi zai kammala , kuma a hirar su da wakilin Muaryar Amurka , Komared Mohammed Jaji ya yi tsokaci game da mataki na gaba .

inputs/Hausa/Example4.txt

@@ -0,0 +1,2 @@
+Kan haka Majalisar Dinkin Duniya ta zabi Aliko Dangote , da shugaban bankin raya Afirka , da wassu m... 
+Kan haka Majalisar Dinkin Duniya ta zabi Aliko Dangote , da shugaban bankin raya Afirka , da wassu mutane 25 a fadin duniya su jagoranci magance matsalar tamowa , kafin shekara 2030 .

inputs/Hausa/Example5.txt

@@ -0,0 +1,2 @@
+Temitope Olatoye Sugar shine mai wakiltar mazabar Lagelu da Akinyele daga jihar Oyo , a majalisar wa... 
+Temitope Olatoye Sugar shine mai wakiltar mazabar Lagelu da Akinyele daga jihar Oyo , a majalisar wakilan tarayyar Najeriya .

inputs/Hausa/Example6.txt

@@ -0,0 +1,2 @@
+Tsohon mataimakin shugaban Najeriya , kuma dan takarar shugaban kasa a zaben 2019 karkashin jam’iyya... 
+Tsohon mataimakin shugaban Najeriya , kuma dan takarar shugaban kasa a zaben 2019 karkashin jam’iyyar adawa ta PDP , Atiku Abubakar , ya yi Allah wadai da yunkurin da wasu sojoji suka yi na kifar da “ zababbiyar gwamnatin Habasha .

inputs/Igbo/Example1.txt

@@ -0,0 +1,2 @@
+Osote onye - isi ndị ome - iwu Naịjirịa bụ Ike Ekweremadu ekwuola na ike agwụla ndị Sịnatị iji otu n... 
+Osote onye - isi ndị ome - iwu Naịjirịa bụ Ike Ekweremadu ekwuola na ike agwụla ndị Sịnatị iji otu nkeji darajụụ akwanyere ndị egburu n'ime oke ọgbaghara dị na Naịjirịa oge ọ bula .

inputs/Igbo/Example2.txt

@@ -0,0 +1,2 @@
+Okwu a Buhari kwuru na isi ndọrọndọrọ ọchịchị na 2015 bu ịhe eji kpụrụ ya na ọnụ ugbua , ọkachasị ka... 
+Okwu a Buhari kwuru na isi ndọrọndọrọ ọchịchị na 2015 bu ịhe eji kpụrụ ya na ọnụ ugbua , ọkachasị ka ụlọ ọrụ na - ahụ maka ọnụ ọgụgụ a na - akpọ National Bureau of Statistics ( NBS ) nwepụtara ozi n'akọwa na mmadụ ruru nde asaa na nari ise so na ndị enweghi ọrụ kemgbe afọ 2016 .

inputs/Igbo/Example3.txt

@@ -0,0 +1,2 @@
+Google Africa kwuru n'igwe okwu Twitter sị : Taa , anyị na - akwanyere onye egwuregwu bọọlụ a ma ama... 
+Google Africa kwuru n'igwe okwu Twitter sị : Taa , anyị na - akwanyere onye egwuregwu bọọlụ a ma ama , Stephen Keshi ugwu .

inputs/Igbo/Example4.txt

@@ -0,0 +1,2 @@
+Keshi chịrị ndị otu egwuregwu Super Eagles kemgbe afọ 2011 ma durukwa ha gaa asọmpi dị iche iche nke... 
+Keshi chịrị ndị otu egwuregwu Super Eagles kemgbe afọ 2011 ma durukwa ha gaa asọmpi dị iche iche nke gụnyere ; Iko Mba Afrika na 2013 ( nke ha bulatara Naịjirịa ) , iko mpaghara Afrịka dị iche iche na 2013 , ma nye aka wetara Naijiria ọnọdụ n'asọmpi Iko Mbaụwa niile na 2014 .

inputs/Igbo/Example5.txt

@@ -0,0 +1,2 @@
+N' akwụkwọ ozi , ngalaba 'US Department' tinyere na websait ha , ha kwuru sị : Yunaited Steeti na - ... 
+N' akwụkwọ ozi , ngalaba 'US Department' tinyere na websait ha , ha kwuru sị : Yunaited Steeti na - enwe obi mwute n' iyi ọrụ nke onye ndu ndị na - ama gọọmenti Kenya aka n'ihu bụ Raila Odinga duru onwe ya ka ọnwa Jenuwari gbara ịrị atọ .

inputs/Igbo/Example6.txt

@@ -0,0 +1,2 @@
+Taa , otu n'ime ndị kewapụtara n'otu ndọrọndọrọ ọchịchị APC kpọrọ ndị ntaakụkọ n'isi ụlọọrụ ha maka ... 
+Taa , otu n'ime ndị kewapụtara n'otu ndọrọndọrọ ọchịchị APC kpọrọ ndị ntaakụkọ n'isi ụlọọrụ ha maka ị kọwa echiche ha n'esomokwu nke di n'etiti ndị APC nke Imo steeti . N'ọnụ okwu TOE Ekechi bụ onụ na - ekwuchitere otu a , ha na - ebo gọvanọ Okorocha ebubo na o nupuru iwu ji patu ha isi ọtụtụ

inputs/Igbo/Example7.txt

@@ -0,0 +1,2 @@
+Otu kporo onweha 'The Coalition of Northern Groups' na bekee gwara onyeisiala Naịjirịa bụ Muhammadu ... 
+Otu kporo onweha 'The Coalition of Northern Groups' na bekee gwara onyeisiala Naịjirịa bụ Muhammadu Buhuri na onye chiburu dịka osote onyeisiala n'oge garaaga bụ Atiku Abubakar na ọ ga - adị mma maọbụrụ na ha abụọ wepuru aka n'ime ọsọ ị banye n'ọkwa ọchịchị dịka onyeisiala n'afọ 2019 .

inputs/Kinyarwanda/Example1.txt

@@ -0,0 +1,2 @@
+Ambasaderi w’Umuryango w’Ubumwe bw’u Burayi mu Rwanda , Nicola Bellomo , aherutse gushima uko u Rwan... 
+Ambasaderi w’Umuryango w’Ubumwe bw’u Burayi mu Rwanda , Nicola Bellomo , aherutse gushima uko u Rwanda rurimo guhangana n’icyorezo cya Coronavirus , yizeza ko uyu muryango uzakomeza gufatanya na rwo muri uru rugamba no mu zindi gahunda z’iterambere .

inputs/Kinyarwanda/Example2.txt

@@ -0,0 +1,2 @@
+Imibare ya Banki y’Isi yo kuwa 9 Mata igaragaza ko ubukungu bwo muri Afurika yo-munsi y’Ubutayu bwa ... 
+Imibare ya Banki y’Isi yo kuwa 9 Mata igaragaza ko ubukungu bwo muri Afurika yo-munsi y’Ubutayu bwa Sahara , bwagizweho ingaruka na Coronavirus ndetse ko buzamanuka ku kigero cya - 2 .

inputs/Kinyarwanda/Example3.txt

@@ -0,0 +1,2 @@
+Amagambo ya Mateke anahura n’ay’umudepite Ruth Nankabirwa , kuri uyu kuwa Gatatu wabwiye bagenzi be ... 
+Amagambo ya Mateke anahura n’ay’umudepite Ruth Nankabirwa , kuri uyu kuwa Gatatu wabwiye bagenzi be mu Nteko Ishinga Amategeko ko Guverinoma ya Uganda ikwiye gukemura bwangu ikibazo ifitanye n’u Rwanda , ariko asa n’uca amarenga ku buryo bwakoreshwa .

inputs/Kinyarwanda/Example4.txt

@@ -0,0 +1,2 @@
+Ubwo bari ku ngingo zijyanye n’uko Uganda ifasha imitwe yitwaje intwaro , Nduhungirehe yatanze uruge... 
+Ubwo bari ku ngingo zijyanye n’uko Uganda ifasha imitwe yitwaje intwaro , Nduhungirehe yatanze urugero rw’igitero cyabaye mu ijoro rishyira ku itariki ya Kane Ukwakira aho abarwanyi b’umutwe wa RUD Urunana bateye mu Kinigi .

inputs/Luganda/Example1.txt

@@ -0,0 +1,2 @@
+Phillip Wokorach , Justin Kimono ne Adrian Kisito be bamu ku baayambye Uganda , eyawangula empala zi... 
+Phillip Wokorach , Justin Kimono ne Adrian Kisito be bamu ku baayambye Uganda , eyawangula empala zino omwaka oguwedde , okuva emabeganefuna obuwanguzi .

inputs/Luganda/Example2.txt

@@ -0,0 +1,2 @@
+Oluvannyuma yaddukira mu Zimbabwe ngakozesa Paasipooti eyali mu mannya ga David Mubiru , kyokka aboo... 
+Oluvannyuma yaddukira mu Zimbabwe ngakozesa Paasipooti eyali mu mannya ga David Mubiru , kyokka aboobuyinza baamuyigga ne bamukomyawo mu Uganda , mu November 2016 , okumalayo ekibonerezo ekyemyaka ena nemyaka emirala ebiri , egyamwongerwako olwokutoloka mu kkomera .

inputs/Luganda/Example3.txt

@@ -0,0 +1,2 @@
+DPC wa Rakai , Patience Baganzi yategeezezza nti bagenda kumukwasa poliisi ye Katwe mu Kampala gye y... 
+DPC wa Rakai , Patience Baganzi yategeezezza nti bagenda kumukwasa poliisi ye Katwe mu Kampala gye yaddiza omusango avunaanibwe .

inputs/Luganda/Example4.txt

@@ -0,0 +1,2 @@
+OMWAMI wa Ssabasajja owessaza lya Mawokota afudde kibwatukira nalekabanna Mawokota mu kiyongobero . ... 
+OMWAMI wa Ssabasajja owessaza lya Mawokota afudde kibwatukira nalekabanna Mawokota mu kiyongobero . Kayima David Ssekyeru afudde mu ngeri yentiisa bwaseredde nagwa mu kinaabiro nga egenze okunaaba bagenze okuyita ambulensi okumuddusa mu ddwaliro e Mmengo nafiira mu kkubo nga tebanatuuka mu ddwaliro . Ssekyeru abadde amaze wiiki emu nga mugonvugonvu kyokka abadde azeemu endasi kwekwewaliriza agende mu kinaabiro

inputs/Luganda/Example5.txt

@@ -0,0 +1,2 @@
+Omwogezi wa poliisi mu Greater Masaka , ASP Paul Kangave yategeezezza Bukedde nti poliisi yatandikid... 
+Omwogezi wa poliisi mu Greater Masaka , ASP Paul Kangave yategeezezza Bukedde nti poliisi yatandikiddewo okunoonyereza oluvannyuma lwokufuna amawulire gokutemulwa kwomusuubuzi ono .

inputs/Nigerian/Example1.txt

@@ -0,0 +1,2 @@
+Jii 2 go mane gin ja apiko moro ma ja higni 20 mane oyang nyinge kaka Kevin Omondi kod achiel kuom j... 
+Jii 2 go mane gin ja apiko moro ma ja higni 20 mane oyang nyinge kaka Kevin Omondi kod achiel kuom jowuoth mage mane oting' o mane iluongo ni Shopie Anyango ma ja higni 23 ne jotho mana kanyo gi kanyo e masirano mane ojuko lori moro mar kambi jo China kod apiko yoo Ringa

inputs/Nigerian/Example2.txt

@@ -0,0 +1,2 @@
+Japuonjreno ma wuoi ma jahigni 15 ochopo e nyim jayal bura Joseph Karanja kama odonjne kod ketho mar... 
+Japuonjreno ma wuoi ma jahigni 15 ochopo e nyim jayal bura Joseph Karanja kama odonjne kod ketho mar nego Noel Adhiambo midenyo ma jahigni 11 ; mane en japuonjre e skul ma Kosele Community Christian Center e kar chung' od bura ma Kasipul dwee mokalo .

inputs/Nigerian/Example3.txt

@@ -0,0 +1,2 @@
+Magi oyangi gi jawach eloo State House nyadendi Kanze Dena mane owacho ni jogo nyocha opim ne tuono ... 
+Magi oyangi gi jawach eloo State House nyadendi Kanze Dena mane owacho ni jogo nyocha opim ne tuono e pimo manyocha otim chieng tich 4

inputs/Nigerian/Example4.txt

@@ -0,0 +1,2 @@
+Kanomedochiwo ler ewii wachno Kanze nowacho ni jotich duto mag State House ipimoga moting' o e kinde... 
+Kanomedochiwo ler ewii wachno Kanze nowacho ni jotich duto mag State House ipimoga moting' o e kinde ka kinde moting' o nyaka jatend piny Kenya migosi Uhuru Kenyata gi familia mare mar ng' eyo chal margi ne tuo mar Covid - 19no kowacho ni jii 4 mane oyudi ni kod tuono sani jonie kar thieth ma Kenyatta University Teaching , Referal and Research Hospital ma gidhiyoe nyime gi yudo thieth

inputs/Pidgin/Example1.txt

@@ -0,0 +1,2 @@
+Popular cable satellite broadcaster DsTV , no get right to Bundesliga live matches for di 2019 / 202... 
+Popular cable satellite broadcaster DsTV , no get right to Bundesliga live matches for di 2019 / 2020 season so na pipo wey get StarTimes dey in luck because na dem get broadcast rights for Sub - Saharan Africa .

inputs/Pidgin/Example2.txt

@@ -0,0 +1,2 @@
+Whichever way wey you watch just know say you dey part of one billion pipo wey Bayern CEO Karl - Hei... 
+Whichever way wey you watch just know say you dey part of one billion pipo wey Bayern CEO Karl - Heinz Rummenigge don gauge say go watch dis weekend live matches See Saturday games .

inputs/Pidgin/Example3.txt

@@ -0,0 +1,2 @@
+Conditions Spain top league and working place of Lionel Messi dey torchlight June 12 as di date when... 
+Conditions Spain top league and working place of Lionel Messi dey torchlight June 12 as di date when dem go resume di season .

inputs/Pidgin/Example4.txt

@@ -0,0 +1,2 @@
+LA Lakers legend Kobe Bryant and im daughter Gianna plus seven oda die for helicopter crash for di c... 
+LA Lakers legend Kobe Bryant and im daughter Gianna plus seven oda die for helicopter crash for di city of Calabasa , California on Sunday 26 January .

inputs/Pidgin/Example5.txt

@@ -0,0 +1,2 @@
+Ighalo move go Chinese Super League for 2017 , first with Changch... 
+Ighalo move go Chinese Super League for 2017 , first with Changchun Yatai .

inputs/Pidgin/Example6.txt

@@ -0,0 +1,2 @@
+Senegal and Liverpool forward Mane beat both Egypt player Mohammed Salah and Algeria winger Riyad Ma... 
+Senegal and Liverpool forward Mane beat both Egypt player Mohammed Salah and Algeria winger Riyad Mahrez to win di award wey dem do for Egypt on Tuesday .

inputs/Swahilu/Example1.txt

@@ -0,0 +1,2 @@
+Wanamgambo wa ADF Mauaji ya Alhamisi katika mkoa wa Mbau kaskazini mwa Beni yanashukiwa kufanya na k... 
+Wanamgambo wa ADF Mauaji ya Alhamisi katika mkoa wa Mbau kaskazini mwa Beni yanashukiwa kufanya na kundi la waasi la Allied Democratic Force , ADF , ambalo linahusika na mfululizo wa mauaji tangu kuanza kwa ghasia mwezi November .

inputs/Swahilu/Example2.txt

@@ -0,0 +1,2 @@
+Jeshi la Congo limegundua ‘kiwanda cha kutengeneza mabomu ya kienyeji’ katika kambi moja ya ADF wali... 
+Jeshi la Congo limegundua ‘kiwanda cha kutengeneza mabomu ya kienyeji’ katika kambi moja ya ADF waliyoiteka , msemaji wa jeshi jenerali Leon Richard Kasonga amesema Jumatano .

inputs/Swahilu/Example3.txt

@@ -0,0 +1,2 @@
+Wajumbe wa kikosi kazi cha virusi vya corona cha White House wamepangiwa kutoa ushuhuda mbele ya kam... 
+Wajumbe wa kikosi kazi cha virusi vya corona cha White House wamepangiwa kutoa ushuhuda mbele ya kamati ya Nishati na Biashara ya Baraza la Wawakilishi Jumanne , na Spika wa Baraza la Wawakilishi Nancy Pelosi amesema , “ Wananchi wa Marekani wanahitaji majibu kwa nini Rais Trump anataka upimaji upunguzwe kasi wakati wataalam wanasema upimaji zaidi unahitajika .

inputs/Swahilu/Example4.txt

@@ -0,0 +1,2 @@
+Siku Jumatano maafisa wawili wa Umoja wa Mataifa watawasilisha ripoti inayoeleza kwamba kuna ushahid... 
+Siku Jumatano maafisa wawili wa Umoja wa Mataifa watawasilisha ripoti inayoeleza kwamba kuna ushahidi wa kutosha unaodhihirisha kwamba Saudi Arabia ilidukua simu ya Bezos .

inputs/Swahilu/Example5.txt

@@ -0,0 +1,2 @@
+Mahakama ya Juu ya Korea Kusini imeamrisha mahakama ya chini ifikirie tena moja ya mashtaka ya jinai... 
+Mahakama ya Juu ya Korea Kusini imeamrisha mahakama ya chini ifikirie tena moja ya mashtaka ya jinai dhidi ya Rais wa zamani Park Geun - hye ambaye alilazimishwa kuondoka madarakani mwaka 2017 kutokana na kashfa ya ufisadi .

inputs/Swahilu/Example6.txt

@@ -0,0 +1,2 @@
+Waziri Mkuu wa Uingereza Boris Johnson amesema ataheshimu utaratibu wa sheria lakini Uingereza itaji... 
+Waziri Mkuu wa Uingereza Boris Johnson amesema ataheshimu utaratibu wa sheria lakini Uingereza itajiondowa kutoka Umoja wa Ulaya ( EU ) ifikapo Oktoba 31 .

inputs/Wolof/Example1.txt

@@ -0,0 +1,2 @@
+Dafa di sax , ni mu ame woon noonu fit moo taxoon ñu dàq ko , moom ak benn doomu Farãs bu daan wuyoo... 
+Dafa di sax , ni mu ame woon noonu fit moo taxoon ñu dàq ko , moom ak benn doomu Farãs bu daan wuyoo ci turu Daniel Cohn - Bendit , ca daara ju mag jooju , ci atum 1969 .

inputs/Wolof/Example2.txt

@@ -0,0 +1,2 @@
+Usmaan Sonkoo ngi juddoo Cees ci a... 
+Usmaan Sonkoo ngi juddoo Cees ci atum 1974 .

inputs/Wolof/Example3.txt

@@ -0,0 +1,2 @@
+Waaw , Isaa Sàll nekkoon na fi Njiitu ndajem diiwaanu Fatig ci njeexitalu a... 
+Waaw , Isaa Sàll nekkoon na fi Njiitu ndajem diiwaanu Fatig ci njeexitalu atiy 1990 .