new: First version

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+images/

Makefile

@@ -0,0 +1,42 @@
+#################################################################################
+# GLOBALS                                                                       #
+#################################################################################
+
+PYTHON_VERSION = python3.8
+VIRTUALENV := .venv
+
+#################################################################################
+# COMMANDS                                                                      #
+#################################################################################
+
+# Set the default location for the virtualenv to be stored
+# Create the virtualenv by installing the requirements and test requirements
+
+$(VIRTUALENV)/.installed: requirements.txt
+	@if [ -d $(VIRTUALENV) ]; then rm -rf $(VIRTUALENV); fi
+	@mkdir -p $(VIRTUALENV)
+	virtualenv --python $(PYTHON_VERSION) $(VIRTUALENV)
+	$(VIRTUALENV)/bin/pip3 install -r requirements.txt
+	$(VIRTUALENV)/bin/pip3 install -r requirements_dev.txt
+	#${VIRTUALENV}/bin/pre-commit install --hook-type pre-push --hook-type post-checkout --hook-type pre-commit
+	touch $@
+
+# Update the requirements to latest. This is required because typically we won't
+# want to incldue test requirements in the requirements of the application, and
+# because it makes life much easier when we want to keep our dependencies up to
+# date.
+
+.PHONY: update-requirements-txt
+update-requirements-txt: unpinned_requirements.txt
+update-requirements-txt: VIRTUALENV := /tmp/update-requirements-virtualenv
+update-requirements-txt:
+	@if [ -d $(VIRTUALENV) ]; then rm -rf $(VIRTUALENV); fi
+	@mkdir -p $(VIRTUALENV)
+	virtualenv --python $(PYTHON_VERSION) $(VIRTUALENV)
+	$(VIRTUALENV)/bin/pip3 install --upgrade -r unpinned_requirements.txt
+	echo "# Created by 'make update-requirements-txt'. DO NOT EDIT!" > requirements.txt
+	$(VIRTUALENV)/bin/pip freeze | grep -v pkg_resources==0.0.0 >> requirements.txt
+
+.PHONY: virtualenv
+virtualenv: $(VIRTUALENV)/.installed
+

app.py

@@ -0,0 +1,40 @@
+import spacy
+import streamlit as st
+
+# from transformers import AutoModelForTokenClassification, AutoTokenizer, pipeline
+
+
+def render_entities(entities):
+    colors = {"LOCATION": "#5cff84"}
+    options = {"ents": ["LOCATION"], "colors": colors}
+    html = spacy.displacy.render(entities, style="ent", options=options, manual=True)
+    html = html.replace("\n", " ")
+
+    return html
+
+
+HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem">{}</div>"""
+
+st.header("Location Entity Recognition Demo 🔎🌆🌍")
+threshold = st.sidebar.slider("Threshold", value=0.5, min_value=0.0, max_value=1.0)
+display_probabilities = st.sidebar.checkbox("Display probabilities")
+
+
+text = st.text_area("Text input", value="This text is about Malaria", height=400)
+
+nlp = spacy.load("en_core_web_trf")
+
+doc = nlp(text)
+
+ents = [
+    {"start": ent.start_char, "end": ent.end_char, "label": "LOCATION"}
+    for ent in doc.ents
+]
+foo = {"text": text, "ents": ents}
+
+
+print(ents)
+print(doc.ents)
+
+html = render_entities(foo)
+st.write(HTML_WRAPPER.format(html), unsafe_allow_html=True)

data/.gitignore

@@ -0,0 +1 @@
+/raw

data/processed/.gitignore

@@ -0,0 +1 @@
+/wellcome_grant_descriptions.csv

data/raw.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: b50ad44187720e74dddd6ec443649e27.dir
+  size: 42921143
+  nfiles: 2
+  path: raw

dvc.lock

@@ -0,0 +1,15 @@
+schema: '2.0'
+stages:
+  subset:
+    cmd: python src/subset_data.py
+    deps:
+    - path: data/raw/Wellcome-grants-awarded-1-October-2005-to-04-05-2022.csv
+      md5: 5c0d0e532709648b61625e7e130dfaa4
+      size: 31028261
+    - path: src/subset_data.py
+      md5: 3b6059867baea4de020776bcfdc9c2a4
+      size: 604
+    outs:
+    - path: data/processed/wellcome_grant_descriptions.csv
+      md5: bb28282adc17ccd209ed370bc4557e40
+      size: 1307583

dvc.yaml

@@ -0,0 +1,8 @@
+stages:
+  subset:
+    cmd: python src/subset_data.py
+    deps:
+      - src/subset_data.py
+      - data/raw/Wellcome-grants-awarded-1-October-2005-to-04-05-2022.csv
+    outs:
+      - data/processed/wellcome_grant_descriptions.csv

requirements.txt

@@ -0,0 +1,79 @@
+# Created by 'make update-requirements-txt'. DO NOT EDIT!
+altair==4.2.0
+attrs==22.1.0
+backports.zoneinfo==0.2.1
+blinker==1.5
+blis==0.7.8
+cachetools==5.2.0
+catalogue==2.0.8
+certifi==2022.9.24
+charset-normalizer==2.1.1
+click==8.1.3
+commonmark==0.9.1
+confection==0.0.3
+cymem==2.0.6
+decorator==5.1.1
+en-core-web-trf==3.4.0
+entrypoints==0.4
+filelock==3.8.0
+gitdb==4.0.9
+GitPython==3.1.29
+huggingface-hub==0.10.1
+idna==3.4
+importlib-metadata==5.0.0
+importlib-resources==5.10.0
+Jinja2==3.1.2
+jsonschema==4.16.0
+langcodes==3.3.0
+MarkupSafe==2.1.1
+murmurhash==1.0.8
+numpy==1.23.3
+packaging==21.3
+pandas==1.5.0
+pathy==0.6.2
+Pillow==9.2.0
+pkgutil-resolve-name==1.3.10
+preshed==3.0.7
+protobuf==3.20.3
+pyarrow==9.0.0
+pydantic==1.9.2
+pydeck==0.8.0b3
+Pygments==2.13.0
+Pympler==1.0.1
+pyparsing==3.0.9
+pyrsistent==0.18.1
+python-dateutil==2.8.2
+pytz==2022.4
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+regex==2022.9.13
+requests==2.28.1
+rich==12.6.0
+semver==2.13.0
+six==1.16.0
+smart-open==5.2.1
+smmap==5.0.0
+spacy==3.4.1
+spacy-alignments==0.8.5
+spacy-legacy==3.0.10
+spacy-loggers==1.0.3
+spacy-transformers==1.1.8
+srsly==2.4.4
+streamlit==1.13.0
+thinc==8.1.3
+tokenizers==0.13.1
+toml==0.10.2
+toolz==0.12.0
+torch==1.12.1
+tornado==6.2
+tqdm==4.64.1
+transformers==4.23.1
+typer==0.4.2
+typing-extensions==4.4.0
+tzdata==2022.4
+tzlocal==4.2
+urllib3==1.26.12
+validators==0.20.0
+wasabi==0.10.1
+watchdog==2.1.9
+zipp==3.9.0

requirements_dev.txt

@@ -0,0 +1,2 @@
+dvc[s3]
+pre-commit

src/subset_data.py

@@ -0,0 +1,25 @@
+import numpy as np
+import pandas as pd
+
+INPUT_FILE = "data/raw/Wellcome-grants-awarded-1-October-2005-to-04-05-2022.csv"
+OUTPUT_FILE = "data/processed/wellcome_grant_descriptions.csv"
+
+print(f"Reading data from {INPUT_FILE}")
+
+
+data = pd.read_csv(INPUT_FILE)
+
+data = (
+    data[["Description"]]
+    .replace("Not available", np.nan)
+    .dropna()
+    .drop_duplicates()
+    .reset_index(drop=True)
+    .sample(1000)
+)
+
+print(f"Number of rows: {data.shape[0]}")
+print(f"Number of unique rows: {data['Description'].nunique()}")
+
+print(f"Saving file to {OUTPUT_FILE}")
+data.to_csv(OUTPUT_FILE, index=False)

unpinned_requirements.txt

@@ -0,0 +1,6 @@
+pandas
+streamlit
+transformers
+torch
+spacy
+https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.4.0/en_core_web_trf-3.4.0-py3-none-any.whl