space

.gitattributes

@@ -1,27 +1,27 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bin.* filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zstandard filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+venv
+test.html

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "workbench.colorCustomizations": {
+        "activityBar.background": "#09323E",
+        "titleBar.activeBackground": "#0C4656",
+        "titleBar.activeForeground": "#F6FCFE"
+    }
+}

README.md

@@ -1,37 +1,37 @@
----
-title: REBEL
-emoji: 🏢
-colorFrom: green
-colorTo: blue
-sdk: streamlit
-app_file: app.py
-pinned: false
----
-
-# Configuration
-
-`title`: _string_
-Display title for the Space
-
-`emoji`: _string_
-Space emoji (emoji-only character allowed)
-
-`colorFrom`: _string_
-Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
-
-`colorTo`: _string_
-Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
-
-`sdk`: _string_
-Can be either `gradio`, `streamlit`, or `static`
-
-`sdk_version` : _string_
-Only applicable for `streamlit` SDK.  
-See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
-
-`app_file`: _string_
-Path to your main application file (which contains either `gradio` or `streamlit` Python code, or `static` html code).
-Path is relative to the root of the repository.
-
-`pinned`: _boolean_
-Whether the Space stays on top of your list.
+---
+title: REBEL
+emoji: 🏢
+colorFrom: green
+colorTo: blue
+sdk: streamlit
+app_file: app.py
+pinned: false
+---
+
+# Configuration
+
+`title`: _string_
+Display title for the Space
+
+`emoji`: _string_
+Space emoji (emoji-only character allowed)
+
+`colorFrom`: _string_
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`colorTo`: _string_
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`sdk`: _string_
+Can be either `gradio`, `streamlit`, or `static`
+
+`sdk_version` : _string_
+Only applicable for `streamlit` SDK.  
+See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
+
+`app_file`: _string_
+Path to your main application file (which contains either `gradio` or `streamlit` Python code, or `static` html code).
+Path is relative to the root of the repository.
+
+`pinned`: _boolean_
+Whether the Space stays on top of your list.

app.py

@@ -1,4 +1,122 @@
+from logging import disable
+from pkg_resources import EggMetadata
 import streamlit as st
+import streamlit.components.v1 as components
+import networkx as nx
+import matplotlib.pyplot as plt
+from pyvis.network import Network
+from streamlit.state.session_state import SessionState
+from streamlit.type_util import Key
+import rebel
+import wikipedia
+
+network_filename = "test.html"
+
+state_variables = {
+    'has_run':False,
+    'wiki_suggestions': "",
+    'wiki_text' : [],
+    'nodes':[]
+}
+
+for k, v in state_variables.items():
+    if k not in st.session_state:
+        st.session_state[k] = v
+
+def clip_text(t, lenght = 5):
+    return ".".join(t.split(".")[:lenght]) + "."
+
+
+
+def generate_graph():
+    if 'wiki_text' not in st.session_state:
+        return
+    if len(st.session_state['wiki_text']) == 0:
+        st.error("please enter a topic and select a wiki page first")
+        return
+    with st.spinner(text="Generating graph..."):
+        texts = st.session_state['wiki_text']
+        nodes = rebel.generate_knowledge_graph(texts, network_filename)
+        st.session_state['nodes'] = nodes
+        st.session_state['has_run'] = True
+    st.success('Done!')
+
+def show_suggestion():
+    with st.spinner(text="fetching wiki topics..."):
+        if st.session_state['input_method'] == "wikipedia":
+            text = st.session_state.text
+            if text is not None:
+                st.session_state['wiki_suggestions'] = wikipedia.search(text, results = 3)
+
+def show_wiki_text(page_title):
+    with st.spinner(text="fetching wiki page..."):
+        try:
+            page = wikipedia.page(title=page_title, auto_suggest=False)
+            st.session_state['wiki_text'].append(clip_text(page.summary))
+        except wikipedia.DisambiguationError as e:
+            with st.spinner(text="Woops, ambigious term, recalculating options..."):
+                st.session_state['wiki_suggestions'].remove(page_title)
+                temp = st.session_state['wiki_suggestions'] + e.options[:3]
+                st.session_state['wiki_suggestions'] = list(set(temp))
+
+def add_text(term):
+    try:
+        extra_text = clip_text(wikipedia.page(title=term, auto_suggest=True).summary)
+        st.session_state['wiki_text'].append(extra_text)
+    except wikipedia.DisambiguationError as e:
+        st.session_state["nodes"].remove(term)
+
+
+def reset_session():
+    for k in state_variables:
+        del st.session_state[k]
+
+st.title('REBELious knowledge graph generation')
+st.session_state['input_method'] = "wikipedia"
+
+# st.selectbox(
+#      'input method',
+#      ('wikipedia', 'free text'),  key="input_method")
+
+if st.session_state['input_method'] != "wikipedia":
+    st.text_area("Your text", key="text")
+else:
+    st.text_input("wikipedia search term",on_change=show_suggestion, key="text")
+
+if len(st.session_state['wiki_suggestions']) != 0:
+    columns = st.columns([1] * len(st.session_state['wiki_suggestions']))
+    for i, (c, s) in enumerate(zip(columns, st.session_state['wiki_suggestions'])):
+        with c:
+            st.button(s, on_click=show_wiki_text, args=(s,), key=i)
+
+if len(st.session_state['wiki_text']) != 0:
+    for t in st.session_state['wiki_text']:
+        new_expander = st.expander(label=t[:30] + "...")
+        with new_expander:
+            st.markdown(t)
+
+if st.session_state['input_method'] != "wikipedia":
+    st.button("find wiki pages")
+    if "wiki_suggestions" in st.session_state:
+            st.button("generate", on_click=generate_graph, key="gen_graph")
+
+else:
+    st.button("generate", on_click=generate_graph, key="gen_graph2")
+
+
+if st.session_state['has_run']:
+    cols = st.columns([4, 1])
+    with cols[0]:
+        HtmlFile = open(network_filename, 'r', encoding='utf-8')
+        source_code = HtmlFile.read()
+        components.html(source_code, height=1500,width=1500)
+    with cols[1]:
+        st.text("expand")
+        for s in st.session_state["nodes"]:
+            st.button(s, on_click=add_text, args=(s,))
+
+
+
+
+
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)

rebel.py

@@ -0,0 +1,119 @@
+from typing import List
+from transformers import pipeline
+from pyvis.network import Network
+from functools import lru_cache
+from app import generate_graph
+import spacy
+from spacy import displacy
+
+
+DEFAULT_LABEL_COLORS = {
+    "ORG": "#7aecec",
+    "PRODUCT": "#bfeeb7",
+    "GPE": "#feca74",
+    "LOC": "#ff9561",
+    "PERSON": "#aa9cfc",
+    "NORP": "#c887fb",
+    "FACILITY": "#9cc9cc",
+    "EVENT": "#ffeb80",
+    "LAW": "#ff8197",
+    "LANGUAGE": "#ff8197",
+    "WORK_OF_ART": "#f0d0ff",
+    "DATE": "#bfe1d9",
+    "TIME": "#bfe1d9",
+    "MONEY": "#e4e7d2",
+    "QUANTITY": "#e4e7d2",
+    "ORDINAL": "#e4e7d2",
+    "CARDINAL": "#e4e7d2",
+    "PERCENT": "#e4e7d2",
+}
+
+def generate_knowledge_graph(texts: List[str], filename: str):
+    nlp = spacy.load("en_core_web_sm")
+    doc = nlp("\n".join(texts))
+    NERs = [ent.text for ent in doc.ents]
+    NER_types =  [ent.label_ for ent in doc.ents]
+    for nr, nrt in zip(NERs, NER_types):
+        print(nr, nrt)
+
+    triplets = []
+    for triplet in texts:
+        triplets.extend(generate_partial_graph(triplet))
+    print(generate_partial_graph.cache_info())
+    heads = [ t["head"] for t in triplets]
+    tails = [ t["tail"] for t in triplets]
+
+    nodes = set(heads + tails)
+    net = Network(directed=True)
+
+    for n in nodes:
+        if n in NERs:
+            NER_type = NER_types[NERs.index(n)]
+            color = DEFAULT_LABEL_COLORS[NER_type]
+            net.add_node(n, title=NER_type, shape="circle", color=color)
+        else:
+            net.add_node(n, shape="circle")
+
+    unique_triplets = set()
+    stringify_trip = lambda x : x["tail"] + x["head"] + x["type"]
+    for triplet in triplets:
+        if stringify_trip(triplet) not in unique_triplets:
+            net.add_edge(triplet["tail"], triplet["head"], title=triplet["type"], label=triplet["type"])
+            unique_triplets.add(stringify_trip(triplet))
+
+    net.repulsion(
+        node_distance=200,
+        central_gravity=0.2,
+        spring_length=200,
+        spring_strength=0.05,
+        damping=0.09
+    )
+    net.set_edge_smooth('dynamic')
+    net.show(filename)
+    return nodes
+
+
+@lru_cache
+def generate_partial_graph(text):
+    triplet_extractor = pipeline('text2text-generation', model='Babelscape/rebel-large', tokenizer='Babelscape/rebel-large')
+    a = triplet_extractor(text, return_tensors=True, return_text=False)[0]["generated_token_ids"]["output_ids"]
+    extracted_text = triplet_extractor.tokenizer.batch_decode(a)
+    extracted_triplets = extract_triplets(extracted_text[0])
+    return extracted_triplets
+
+
+def extract_triplets(text):
+    """
+    Function to parse the generated text and extract the triplets
+    """
+    triplets = []
+    relation, subject, relation, object_ = '', '', '', ''
+    text = text.strip()
+    current = 'x'
+    for token in text.replace("<s>", "").replace("<pad>", "").replace("</s>", "").split():
+        if token == "<triplet>":
+            current = 't'
+            if relation != '':
+                triplets.append({'head': subject.strip(), 'type': relation.strip(),'tail': object_.strip()})
+                relation = ''
+            subject = ''
+        elif token == "<subj>":
+            current = 's'
+            if relation != '':
+                triplets.append({'head': subject.strip(), 'type': relation.strip(),'tail': object_.strip()})
+            object_ = ''
+        elif token == "<obj>":
+            current = 'o'
+            relation = ''
+        else:
+            if current == 't':
+                subject += ' ' + token
+            elif current == 's':
+                object_ += ' ' + token
+            elif current == 'o':
+                relation += ' ' + token
+    if subject != '' and relation != '' and object_ != '':
+        triplets.append({'head': subject.strip(), 'type': relation.strip(),'tail': object_.strip()})
+
+    return triplets
+

requirements.txt

@@ -0,0 +1,125 @@
+altair==4.2.0
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+astor==0.8.1
+attrs==21.4.0
+backcall==0.2.0
+backports.zoneinfo==0.2.1
+base58==2.1.1
+beautifulsoup4==4.10.0
+bleach==4.1.0
+blinker==1.4
+blis==0.7.5
+cachetools==5.0.0
+catalogue==2.0.6
+certifi==2021.10.8
+cffi==1.15.0
+charset-normalizer==2.0.10
+click==7.1.2
+cycler==0.11.0
+cymem==2.0.6
+debugpy==1.5.1
+decorator==5.1.1
+defusedxml==0.7.1
+en-core-web-sm==3.2.0
+entrypoints==0.3
+filelock==3.4.2
+fonttools==4.28.5
+gitdb==4.0.9
+GitPython==3.1.26
+got==0.0.1
+huggingface-hub==0.2.1
+idna==3.3
+importlib-resources==5.4.0
+ipykernel==6.6.1
+ipython==7.31.0
+ipython-genutils==0.2.0
+ipywidgets==7.6.5
+jedi==0.18.1
+Jinja2==3.0.3
+joblib==1.1.0
+jsonpickle==2.0.0
+jsonschema==4.3.3
+jupyter-client==7.1.0
+jupyter-core==4.9.1
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.0.2
+kiwisolver==1.3.2
+langcodes==3.3.0
+MarkupSafe==2.0.1
+matplotlib==3.5.1
+matplotlib-inline==0.1.3
+mistune==0.8.4
+murmurhash==1.0.6
+nbclient==0.5.9
+nbconvert==6.4.0
+nbformat==5.1.3
+nest-asyncio==1.5.4
+networkx==2.6.3
+notebook==6.4.6
+numpy==1.22.0
+packaging==21.3
+pandas==1.3.5
+pandocfilters==1.5.0
+parso==0.8.3
+pathy==0.6.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.0.0
+preshed==3.0.6
+prometheus-client==0.12.0
+prompt-toolkit==3.0.24
+protobuf==3.19.3
+ptyprocess==0.7.0
+pyarrow==6.0.1
+pycparser==2.21
+pydantic==1.8.2
+pydeck==0.7.1
+Pygments==2.11.2
+Pympler==1.0.1
+pyparsing==3.0.6
+pyrsistent==0.18.0
+python-dateutil==2.8.2
+pytz==2021.3
+pytz-deprecation-shim==0.1.0.post0
+pyvis==0.1.9
+PyYAML==6.0
+pyzmq==22.3.0
+regex==2021.11.10
+requests==2.27.1
+sacremoses==0.0.47
+Send2Trash==1.8.0
+six==1.16.0
+smart-open==5.2.1
+smmap==5.0.0
+soupsieve==2.3.1
+spacy==3.2.1
+spacy-legacy==3.0.8
+spacy-loggers==1.0.1
+srsly==2.4.2
+streamlit==1.3.1
+terminado==0.12.1
+testpath==0.5.0
+thinc==8.0.13
+tokenizers==0.10.3
+toml==0.10.2
+toolz==0.11.2
+torch==1.10.1
+tornado==6.1
+tqdm==4.62.3
+traitlets==5.1.1
+transformers==4.15.0
+typer==0.4.0
+typing-extensions==4.0.1
+tzdata==2021.5
+tzlocal==4.1
+urllib3==1.26.8
+validators==0.18.2
+wasabi==0.9.0
+watchdog==2.1.6
+wcwidth==0.2.5
+webencodings==0.5.1
+widgetsnbextension==3.5.2
+wikipedia==1.4.0
+zipp==3.7.0
+https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz#egg=en_core_web_sm