Migrate to HF Space

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 howard-haowen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,29 @@
----
-title: Formosan Languages Haowenchiang
-emoji: 🐨
-colorFrom: purple
-colorTo: indigo
-sdk: streamlit
-sdk_version: 1.10.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/howard-haowen/Formosan-languages/HEAD)
+
+# 台灣南島語-華語句庫資料集
+(Dataset of Formosan-Mandarin sentence pairs)
+
+[點我](https://share.streamlit.io/howard-haowen/formosan-languages/main/app.py)進入互動式查詢系統
+
+## 資料概要
+- 🎢 資料集合計約13萬筆台灣南島語-華語句對
+- ⚠️ 此查詢系統僅供教學與研究之用，內容版權歸原始資料提供者所有
+- 💻 隨機顯示10筆資料
+![data_sample](sample-dataframe.png)
+
+## 資料來源
+- 以下資料經由網路爬蟲取得。
+   + 🥅 九階教材: [族語E樂園](http://web.klokah.tw)
+   + 💬 生活會話: [族語E樂園](http://web.klokah.tw)
+   + 🧗 句型: [族語E樂園](http://web.klokah.tw)
+   + 🔭 文法: [臺灣南島語言叢書](https://alilin.apc.gov.tw/tw/)
+- 詞典資料使用`PDFMiner` 將2019版的PDF檔轉成HTML，再用`BeautifulSoup`抓取句對，偶爾會出現族語跟華語對不上的情形。若發現錯誤，請[聯絡我📩](https://github.com/howard-haowen)。詞典中重複出現的句子已從資料集中刪除。
+   + 📚 詞典: [原住民族語言線上詞典](https://e-dictionary.apc.gov.tw/Index.htm?fbclid=IwAR18XBJPj2xs7nhpPlIUZ-P3joQRGXx22rbVcUvp14ysQu6SdrWYvo7gWCc)
+
+## 統計報告
+- 💻 點擊下面的預覽圖即可進入統計報告互動式查看頁面。報告中新增`word_counts`欄位，計算族語句子的字數。
+
+[![pandas-profile](Pandas-profile-screenshot.png)](https://howard-haowen.github.io/Formosan-languages/Pandas-profile-report-of-the-dataset.html)
+
+***
+![](https://octodex.github.com/images/yaktocat.png)

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-leap-day

app.py

@@ -0,0 +1,146 @@
+import pandas as pd
+import streamlit as st
+import streamlit.components.v1 as components
+import re
+from pandas_profiling import ProfileReport
+
+def main():
+  st.title("台灣南島語-華語句庫資料集")
+  st.subheader("Dataset of Formosan-Mandarin sentence pairs")
+  st.markdown(
+    """
+![visitors](https://visitor-badge.glitch.me/badge?page_id=howard-haowen.Formosan-languages)
+
+### 資料概要
+- 🎢 資料集合計約13萬筆台灣南島語-華語句對
+- ⚠️ 此查詢系統僅供教學與研究之用，內容版權歸原始資料提供者所有
+
+### 資料來源
+- 以下資料經由網路爬蟲取得。
+   + 🥅 九階教材: [族語E樂園](http://web.klokah.tw)
+   + 💬 生活會話: [族語E樂園](http://web.klokah.tw)
+   + 🧗 句型: [族語E樂園](http://web.klokah.tw)
+   + 🔭 文法: [臺灣南島語言叢書](https://alilin.apc.gov.tw/tw/)
+- 詞典資料使用`PDFMiner` 將2019版的PDF檔轉成HTML，再用`BeautifulSoup`抓取句對，偶爾會出現族語跟華語對不上的情形。若發現錯誤，請[聯絡我📩](https://github.com/howard-haowen)。詞典中重複出現的句子已從資料集中刪除。
+   + 📚 詞典: [原住民族語言線上詞典](https://e-dictionary.apc.gov.tw/Index.htm?fbclid=IwAR18XBJPj2xs7nhpPlIUZ-P3joQRGXx22rbVcUvp14ysQu6SdrWYvo7gWCc)
+
+### 查詢方法
+- 🔭 過濾：使用左側欄功能選單可過濾資料來源(可多選)與語言，也可使用華語或族語進行關鍵詞查詢。
+  - 🔍 關鍵詞查詢支援[正則表達式](https://zh.wikipedia.org/zh-tw/正则表达式)。
+  - 🥳 族語範例: 
+    + 使用`cia *`查詢布農語，能找到包含`danumcia`、`luduncia`或`siulcia`等詞的句子。
+    + 使用`[a-z]{15,}`查詢任何族語，能找到包含15個字母以上單詞的句子，方便過濾長詞。
+  - 🤩 華語範例: 
+    + 使用`^有一`查詢華語，能找到使用`有一天`、`有一塊`或`有一晚`等詞出現在句首的句子。
+    + 使用`[0-9]{1,}`查詢華語，能找到包含羅馬數字的句子，如`我今年16歲了`。
+- 📚 排序：點選標題列。例如點選`族語`欄位標題列內的任何地方，資料集便會根據族語重新排序。
+- 💬 更多：文字長度超過欄寬時，將滑鼠滑到欄位上方即可顯示完整文字。
+- 🥅 放大：點選表格右上角↘️進入全螢幕模式，再次點選↘️返回主頁。
+
+"""
+)
+  # fetch the raw data
+  df = get_data()
+  # pd.set_option('max_colwidth', 600)
+  
+  # remap column names
+  zh_columns = {'Lang_En': 'Language','Lang_Ch': '語言_方言', 'Ab': '族語', 'Ch': '華語', 'From': '來源'}
+  df.rename(columns=zh_columns, inplace=True)
+  
+  # set up filtering options
+  source_set = df['來源'].unique()
+  sources = st.sidebar.multiselect(
+        "請選擇資料來源",
+        options=source_set,
+        default='詞典',)
+  langs = st.sidebar.selectbox(
+        "請選擇語言",
+        options=['布農','阿美','撒奇萊雅','噶瑪蘭','魯凱','排灣','卑南',
+                 '泰雅','賽德克','太魯閣','鄒','拉阿魯哇','卡那卡那富',
+                 '邵','賽夏','達悟'],)
+  texts = st.sidebar.radio(
+        "請選擇關鍵詞查詢文字類別",
+        options=['華語','族語'],)
+    
+  # filter by sources
+  s_filt = df['來源'].isin(sources)
+  
+  # select a language 
+  if langs == "噶瑪蘭":
+    l_filt = df['Language'] == "Kavalan"
+  elif langs == "阿美":
+    l_filt = df['Language'] == "Amis"
+  elif langs == "撒奇萊雅":
+    l_filt = df['Language'] == "Sakizaya"
+  elif langs == "魯凱":
+    l_filt = df['Language'] == "Rukai"
+  elif langs == "排灣":
+    l_filt = df['Language'] == "Paiwan"
+  elif langs == "卑南":
+    l_filt = df['Language'] == "Puyuma"
+  elif langs == "賽德克":
+    l_filt = df['Language'] == "Seediq"
+  elif langs == "邵":
+    l_filt = df['Language'] == "Thao"
+  elif langs == "拉阿魯哇":
+    l_filt = df['Language'] == "Saaroa"
+  elif langs == "達悟":
+    l_filt = df['Language'] == "Yami"
+  elif langs == "泰雅":
+    l_filt = df['Language'] == "Atayal"
+  elif langs == "太魯閣":
+    l_filt = df['Language'] == "Truku"
+  elif langs == "鄒":
+    l_filt = df['Language'] == "Tsou"
+  elif langs == "卡那卡那富":
+    l_filt = df['Language'] == "Kanakanavu"
+  elif langs == "賽夏":
+    l_filt = df['Language'] == "Saisiyat"
+  elif langs == "布農":
+    l_filt = df['Language'] == "Bunun"
+  
+  # create a text box for keyword search
+  text_box = st.sidebar.text_input('在下方輸入華語或族語，按下ENTER後便會自動更新查詢結果')
+
+  # search for keywords in Mandarin or Formosan 
+  t_filt = df[texts].str.contains(text_box, flags=re.IGNORECASE)
+  
+  # filter the data based on all criteria
+  filt_df = df[(s_filt)&(l_filt)&(t_filt)]
+  
+  st.markdown(
+    """
+### 查詢結果
+"""
+)
+  # display the filtered data
+  st.dataframe(filt_df, 800, 400)
+ 
+  st.markdown(
+    """
+### 資料統計
+"""
+)
+  # display a data profile report
+  report = get_report()
+  components.html(report, width=800, height=800, scrolling=True)  
+  
+# Cache the raw data and profile report to speed up subseuqent requests 
+@st.cache
+def get_data():
+  df = pd.read_pickle('Formosan-Mandarin_sent_pairs_139023entries.pkl')
+  df = df.astype(str, errors='ignore')
+  df = df.applymap(lambda x: x[1:] if x.startswith(".") else x)
+  df = df.applymap(lambda x: x.strip())
+  filt = df.Ch.apply(len) < 5
+  df = df[~filt]
+  return df
+
+@st.cache
+def get_report():
+  df = get_data()
+  report = ProfileReport(df, title='Report', minimal=True).to_html()
+  return report
+
+if __name__ == '__main__':
+  main()

feature-logs.txt

@@ -0,0 +1,3 @@
+2021-01-05 Added search by input querries, which support regex
+2020-12-21 Added pandas-profiling 
+2020-12-17 Deployed Streamlit app

requirements.txt

@@ -0,0 +1,4 @@
+streamlit
+pandas
+pandas_profiling 
+jupyterlab-git