track data

.gitattributes

@@ -25,3 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/*.csv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,107 @@
+import pandas as pd
+import streamlit as st
+
+from src.config import APP_PAGE_HEADER
+from src.utils import search_df
+
+APP_PAGE_HEADER()
+
+
+@st.cache
+class LoadData:
+    train: pd.DataFrame = pd.read_csv("data/train.csv")
+    train = train.sample(frac=1).reset_index(drop=True)  # shuffle data
+
+    test: pd.DataFrame = pd.read_csv("data/test.csv")
+    titles: pd.DataFrame = pd.read_csv("data/titles.csv")
+
+    # add code titles to train data
+    merged = train.merge(titles, left_on="context", right_on="code")
+    train_df = merged[['id', 'code', 'anchor', 'context', 'target', 'title', 'score']].copy()
+
+    # add relations / edges for knowledge graph
+    train_kg: pd.DataFrame = train_df.copy()
+    train_kg['relation'] = train_kg['code'] + " || " + train_kg['title'] + " || " + train_kg['score'].astype(str)
+
+
+class App:
+    def __init__(self):
+        self.data = LoadData()
+
+    def run(self, debug=False):
+        self.render_header(debug)
+        self.render_body(debug)
+        self.render_footer(debug)
+
+    def render_header(self, *args, **kwargs):
+        pass
+
+    def render_body(self, *args, **kwargs):
+        data = self.data.train
+        st.write(f"> Train data `{data.shape[0]}` rows")
+        filter_ = st.text_input("search", "")
+        if filter_:
+            data = search_df(self.data.train, filter_)
+        st.write(data)
+
+        Helper().visualize()
+
+    def render_footer(self, *args, **kwargs):
+        pass
+
+
+class Helper(App):
+    def visualize(self, *args, **kwargs):
+
+        # filter data for visualization
+        MAX_EDGES = 100
+        sample = self.data.train_kg[:MAX_EDGES]
+
+        st1, st2 = st.columns(2)
+
+        data = self.data.train_kg
+
+        score = st1.selectbox("visualize by score", [""] + data["score"].unique().tolist())
+        if score:
+            sample = data[data["score"] == float(score)][:MAX_EDGES]
+
+        filter_ = st2.text_input("search term to visualize matching phrases")
+        if filter_:
+            sample = search_df(data, filter_)[:MAX_EDGES]
+
+        # create graph
+        nodes = list(sample["anchor"].unique()) + list(sample["target"].unique())
+        edges = [(h, t) for h, t in zip(sample["anchor"].tolist(), sample["target"].tolist())]
+        labels = sample["relation"].tolist()
+        edge_labels = dict(zip(edges, labels))
+
+        # create PyVis network from the graph data
+        self.pyvis_network(nodes, edge_labels)
+        st.write(f"> sample size: {sample.shape[0]}")
+
+    def pyvis_network(self, nodes, edge_labels):
+
+        from stvis import pv_static
+
+        g = self.build_network(edge_labels, nodes)
+
+        pv_static(g)
+
+    @staticmethod
+    @st.experimental_singleton
+    def build_network(edge_labels, nodes):
+        # src: https://stackoverflow.com/a/67279471/2839786
+        from pyvis.network import Network
+        g = Network(height="800px", width="1400px", heading="U.S. Patent Phrase/Context Network", bgcolor="#bbbffz")  # notebook=True,
+        for node in nodes:
+            g.add_node(node)
+        for e in edge_labels:
+            n1, n2 = e[0], e[1]
+            label = edge_labels[e]
+            g.add_edge(n1, n2, title=label, show_edge_weights=True)  # weight 42
+        return g
+
+
+if __name__ == "__main__":
+    app = App()
+    app.run(debug=True)

data/test.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a21727914b70c69f48a1aec1bb7e7c4d7f01adf1ae73f0c77e7a2b62dc6a1de
+size 1965

data/titles.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f138d6bdf2939ba576b96b633d81366123b5f64b9842f567360fb1f9e86a5ace
+size 21605031

data/train.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b233317683bfab1dcc949ed4055f9ed168a26b9ef2b0a7cc0091a6bfbb5139da
+size 2141136

requirements.txt

@@ -0,0 +1,4 @@
+pandas
+streamlit
+pyvis
+stvis

src/config.py

@@ -0,0 +1,15 @@
+import streamlit as st
+
+
+def APP_PAGE_HEADER():
+    st.set_page_config(
+        page_title="U.S. Patent", page_icon="🔬", layout="wide", initial_sidebar_state="collapsed"
+    )
+
+    hide_streamlit_style = """
+                    <style>
+                    #MainMenu {visibility: hidden;}
+                    footer {visibility: hidden;}
+                    </style>
+                    """
+    st.markdown(hide_streamlit_style, unsafe_allow_html=True)

src/utils.py

@@ -0,0 +1,7 @@
+import pandas as pd
+import numpy as np
+
+
+def search_df(df: pd.DataFrame, substring: str, case: bool = False) -> pd.DataFrame:
+    mask = np.column_stack([df[col].astype(str).str.contains(substring.lower(), case=case, na=False) for col in df])
+    return df.loc[mask.any(axis=1)]