il sondaggio che segue fa parte di una ricerca relativa al\n"
+ ]
+ }
+ ],
+ "source": [
+ "with open(\"resources/questionnaire/intro_msg_it_1q.html\", encoding=\"utf-8\") as f:\n",
+ " html_intro = f.read()\n",
+ " print(html_intro[:100])\n",
+ "\n",
+ "intro_text_place = driver.find_element(by=\"xpath\", value=\"//div[@class='QuestionText']\")\n",
+ "# assert \"Click to write\" in intro_text_place.text\n",
+ "intro_text_place.click()\n",
+ "\n",
+ "driver.find_element(by=\"xpath\", value=\"//a[@class='Button Blue CodeButton Disabled']\").click()\n",
+ "# driver.execute_script(f\"document.getElementById(\\\"InlineEditorElement\\\").value = \\\"{html_intro}\\\"\")\n",
+ "pc.copy(html_intro) # --> won't work in notebook\n",
+ "text_editor = driver.find_element(by=\"id\", value=\"InlineEditorElement\")\n",
+ "text_editor.send_keys(Keys.CONTROL, \"a\")\n",
+ "text_editor.send_keys(Keys.BACKSPACE)\n",
+ "text_editor.send_keys(Keys.CONTROL, \"v\")\n",
+ "# text_editor.send_keys(html_intro) # slower but works in notebook"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "* delete old files"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "driver.get(\"https://rug.eu.qualtrics.com/Q/MyProjectsSection\")\n",
+ "_ = ui.WebDriverWait(driver, 30).until(lambda driver: driver.find_elements(by=\"xpath\", value=\"//span\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "to_delete = [span for span in driver.find_elements(by=\"xpath\", value=\"//span[starts-with(@class, 'ResourceRow_projectNameLabel')]\") if span.text.startswith(\"femicides_questionnaire_version\")]\n",
+ "to_delete.extend([span for span in driver.find_elements(by=\"xpath\", value=\"//span[starts-with(@class, 'ResourceRow_projectNameLabel')]\") if span.text.strip().endswith(\"_draft\")])\n",
+ "# to_delete.extend([span for span in driver.find_elements(by=\"xpath\", value=\"//span[starts-with(@class, 'ResourceRow_projectNameLabel')]\") if span.text.strip() == \"questionnaire_final\"])\n",
+ "# to_delete.extend([span for span in driver.find_elements(by=\"xpath\", value=\"//span[starts-with(@class, 'ResourceRow_projectNameLabel')]\") if span.text.strip() == \"Untitled project\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for span in to_delete:\n",
+ " tr = span.find_element(by=\"xpath\", value=\"./../../..\")\n",
+ " options_btn = tr.find_element(by=\"xpath\", value=\".//button[starts-with(@id, 'options-')]\")\n",
+ " options_btn.click()\n",
+ " del_button = driver.find_element(by=\"xpath\", value=\"//div[starts-with(@class, 'ActionsMenu_actions-menu-item') and text()='Delete project']\")\n",
+ " del_button.click()\n",
+ " driver.find_element(by=\"xpath\", value=\"//label[@data-testid='project-action-delete-check-0']\").click()\n",
+ " driver.find_element(by=\"xpath\", value=\"//button[@data-testid='project-action-delete-submit']\").click()\n",
+ "\n",
+ " ui.WebDriverWait(driver, 60).until_not(lambda driver: driver.find_elements(by=\"id\", value=\"project-action-delete\"))"
+ ]
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "200f5294b1c9404b01f49351b46f312ccd3edc553908cd778ad96075cc67dce1"
+ },
+ "kernelspec": {
+ "display_name": "Python 3.7.9 64-bit ('sociofillmore': conda)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.9"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/sociofillmore/scoring/extract_frames.py b/sociofillmore/scoring/extract_frames.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bab12e46a9f545bd7f02b1da78f07ef0a6fade9
--- /dev/null
+++ b/sociofillmore/scoring/extract_frames.py
@@ -0,0 +1,49 @@
+from collections import defaultdict
+import json
+
+import pandas as pd
+
+from sociofillmore.common.analyze_text import analyze_single_document, load_deep_frames_cache
+
+
+def main():
+ frames_of_interest = ["Killing", "Death", "Dead_or_alive", "Event", "Catastrophe", "Undergoing"]
+
+ texts_df = pd.read_csv(
+ "output/femicides/split_data/rai/split_main.texts.meta.csv")
+ deep_frames_cache = load_deep_frames_cache()
+
+ fcp_to_sentences = defaultdict(list)
+
+ for i, (_, row) in enumerate(texts_df.iterrows()):
+ if i % 100 == 0:
+ print(i)
+
+ doc_analysis = analyze_single_document(row["text_id"], row["event_id"], "lome_0shot",
+ "femicides/rai_main", texts_df, deep_frames_cache)
+ for i, sent_analysis in enumerate(doc_analysis):
+ sentence = " ".join(sent_analysis["sentence"])
+
+ for fn_st in sent_analysis["fn_structures"]:
+ frame = fn_st["frame"]
+ tgt_idx = str(fn_st["target"]["tokens_idx"][0])
+ if frame in frames_of_interest:
+ construction = sent_analysis["syntax"][tgt_idx][0]["syn_construction"]
+ fcp_to_sentences[f"{frame}++{construction}"].append({
+ "event_id": row["event_id"],
+ "frame": frame,
+ "construction": construction,
+ "target": fn_st["target"]["tokens_str"],
+ "text_id": row["text_id"],
+ "sentence_idx": i,
+ "sentence_str": sentence,
+ "selected_frame": frame,
+ "selected_cx": construction
+ })
+
+ with open("output/scoring/extracted_frames.json", "w") as f:
+ json.dump(fcp_to_sentences, f, indent=4, sort_keys=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/sociofillmore/webapp/__init__.py b/sociofillmore/webapp/__init__.py
new file mode 100755
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/sociofillmore/webapp/__init__.pyc b/sociofillmore/webapp/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e0d3f1206f6db0592f29e0449ae4582b5826450e
Binary files /dev/null and b/sociofillmore/webapp/__init__.pyc differ
diff --git a/sociofillmore/webapp/__pycache__/__init__.cpython-311.pyc b/sociofillmore/webapp/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b8c3b9912751a8f0207bba12ba44dd67888d9e6e
Binary files /dev/null and b/sociofillmore/webapp/__pycache__/__init__.cpython-311.pyc differ
diff --git a/sociofillmore/webapp/__pycache__/__init__.cpython-37.pyc b/sociofillmore/webapp/__pycache__/__init__.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..33169c2cc87424168cb676a15567400b67539304
Binary files /dev/null and b/sociofillmore/webapp/__pycache__/__init__.cpython-37.pyc differ
diff --git a/sociofillmore/webapp/__pycache__/__init__.cpython-39.pyc b/sociofillmore/webapp/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..acaf5e9cda1d7d1934e61d5e068bd2ba7ad8e45f
Binary files /dev/null and b/sociofillmore/webapp/__pycache__/__init__.cpython-39.pyc differ
diff --git a/sociofillmore/webapp/__pycache__/app.cpython-311.pyc b/sociofillmore/webapp/__pycache__/app.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..11b10a76407095379fb4ecd6ff24a727e9ccb3d4
Binary files /dev/null and b/sociofillmore/webapp/__pycache__/app.cpython-311.pyc differ
diff --git a/sociofillmore/webapp/__pycache__/app.cpython-37.pyc b/sociofillmore/webapp/__pycache__/app.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..58a1c485137ea436dec69d3effa71f7e1bc2a553
Binary files /dev/null and b/sociofillmore/webapp/__pycache__/app.cpython-37.pyc differ
diff --git a/sociofillmore/webapp/__pycache__/app.cpython-39.pyc b/sociofillmore/webapp/__pycache__/app.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ea08f276370998b195335c09383f8edabac39d35
Binary files /dev/null and b/sociofillmore/webapp/__pycache__/app.cpython-39.pyc differ
diff --git a/sociofillmore/webapp/app.py b/sociofillmore/webapp/app.py
new file mode 100755
index 0000000000000000000000000000000000000000..586ba05b8aeba2bea0a51b1ed3058c13dff6fdef
--- /dev/null
+++ b/sociofillmore/webapp/app.py
@@ -0,0 +1,1533 @@
+import dataclasses
+import io
+import json
+import os
+import sys
+import glob
+import random
+import re
+import tarfile
+import datetime
+from collections import defaultdict
+from typing import Dict, List, Optional
+import functools
+import requests
+import tempfile
+
+import lxml.etree as ET
+
+import pandas as pd
+import numpy as np
+import gensim
+import spacy
+
+import nltk
+from nltk.corpus import framenet as fn
+from nltk.corpus.reader.framenet import FramenetError
+
+from flask import Flask, request, render_template, jsonify, redirect, abort, session, url_for
+
+from sociofillmore.common.analyze_text import (
+ FrameStructure,
+ get_syntax_info,
+ is_at_root,
+ process_prediction_file,
+ POSSIBLE_CONSTRUCTIONS,
+ SYNTAX_ANALYSIS_CACHE_FILES,
+ enrich_texts_df,
+ read_frames_of_interest,
+ load_deep_frames_cache,
+ get_text_meta,
+ analyze_single_document,
+ get_tarball_blocks,
+ analyze_external_file
+)
+from sociofillmore.crashes.utils import is_a_dutch_text
+
+
+# download nltk packages if needed
+if sys.argv[2] != "local":
+ nltk.download("framenet_v17", download_dir="/nltk_data")
+ nltk.download("punkt", download_dir="/nltk_data")
+
+# security (very basic!)
+PROTECTED_DATASETS = [] # "femicides/rai"
+if os.path.exists("secrets.json"):
+ with open("secrets.json", encoding="utf-8") as f:
+ secrets = json.load(f)
+ AUTH_KEY = secrets["auth_key"]
+ PASSWORD = secrets["password"]
+ SECRET_KEY = bytes(secrets["flask_secret_key"], "utf-8")
+else:
+ AUTH_KEY = os.environ.get("AUTH_KEY")
+ PASSWORD = os.environ.get("PASSWORD")
+ SECRET_KEY = os.environ.get("FLASK_SECRET_KEY")
+
+# global app object
+app = Flask(__name__)
+app.secret_key = SECRET_KEY
+# app.config["SESSION_PERMANENT"] = False
+# app.config["SESSION_TYPE"] = "filesystem"
+# Session(app)
+
+# gensim & spacy models
+def load_gensim_model(limit):
+ print("Loading GENSIM model... [this can take a few minutes]")
+ return gensim.models.word2vec.KeyedVectors.load_word2vec_format("data/embeddings/concat_glove_frames.w2v.txt", limit=limit)
+
+gensim_m = load_gensim_model(100_000)
+
+print("Loading SpaCy models...")
+spacy_model_ud = spacy.load("xx_sent_ud_sm")
+spacy_model_langs = {
+ "it": spacy.load("it_core_news_md"),
+ "nl": spacy.load("nl_core_news_md"),
+ "en": spacy.load("en_core_web_md")
+}
+
+
+
+# frequency information cache
+frame_freq_cache = {}
+
+with open("resources/fn_frames_to_roles.json", encoding="utf-8") as f:
+ fn_frames_to_roles = json.load(f)
+
+# data processing constants
+VICTIM_AGE_GROUPS = ["0-12", "12-18", "18-30", "30-50", "50-70", "70-120"]
+ALL_FOREIGN_NATIONALITIES = "estero (tutto)"
+
+
+deep_frames_cache = load_deep_frames_cache()
+
+def read_rai_provider_attrs():
+ df = pd.read_excel("resources/RAI_sources_mr.xlsx")
+
+ return {
+ "politics:man:left": df[df["politics_man"] == "L"]["source"].unique().tolist(),
+ "politics:tc:left": df[df["politics_tc_cat"] == "L"]["source"]
+ .unique()
+ .tolist(),
+ "politics:agg:left": df[df["politics_agg"] == "L"]["source"].unique().tolist(),
+ "politics:man:right": df[df["politics_man"] == "R"]["source"].unique().tolist(),
+ "politics:tc:right": df[df["politics_tc_cat"] == "R"]["source"]
+ .unique()
+ .tolist(),
+ "politics:agg:right": df[df["politics_agg"] == "R"]["source"].unique().tolist(),
+ "politics:man:neutral": df[df["politics_man"] == "N"]["source"]
+ .unique()
+ .tolist(),
+ "politics:tc:neutral": df[df["politics_tc_cat"] == "N"]["source"]
+ .unique()
+ .tolist(),
+ "politics:agg:neutral": df[df["politics_agg"] == "N"]["source"]
+ .unique()
+ .tolist(),
+ "type:agency": df[df["type"] == "A"]["source"].unique().tolist(),
+ "type:outlet": df[df["type"] == "OUTLET"]["source"].unique().tolist(),
+ "area:national": df[df["area"] == "nat"]["source"].unique().tolist(),
+ "area:regional": df[df["area"] == "loc"]["source"].unique().tolist(),
+ }
+
+def read_migration_provider_attrs():
+ df = pd.read_csv("data/migration/provider_pol_rel_ratings.csv")
+
+ return {
+ "politics:sc:left": df[df["political_stance"] == -1]["provider"].unique().tolist(),
+ "politics:sc:right": df[df["political_stance"] == 1]["provider"].unique().tolist(),
+ "politics:sc:neutral": df[df["political_stance"] == 0]["provider"].unique().tolist(),
+ "religion:sc:catholic": df[df["religious"] == True]["provider"].unique().tolist(),
+ "religion:sc:non_catholic": df[df["religious"] == False]["provider"].unique().tolist()
+ }
+
+
+def read_crashes_provider_attrs():
+ df = pd.read_csv("resources/crashes_sources.csv")
+
+ # remove empty rows
+ df = df.dropna(subset=["ProviderNameCorr"])
+
+ # map to correct names
+ name_map = {
+ row["ProviderName"]: row["ProviderNameCorr"].strip('"')
+ for _, row in df.iterrows()
+ }
+
+ # merge duplicates
+ df = df.groupby(list(df.columns[2:11]))["ProviderFreq"].apply(sum).reset_index()
+
+ # "explode" multiple province fields
+ df = df.assign(**{"Province": df["Province"].str.split("|")}).explode("Province")
+
+ attr_map = {
+ f"{col}:{val}": df[df[col] == val]["ProviderNameCorr"].unique().tolist()
+ for col in df.columns[1:9]
+ for val in set(df[col].values)
+ if val != "-"
+ }
+
+ return attr_map, name_map
+
+PROVIDER_ATTRS = {
+ "femicides/rai": read_rai_provider_attrs(),
+ "femicides/olv": {},
+ "crashes/thecrashes": read_crashes_provider_attrs()[0],
+ "migration/pavia": read_migration_provider_attrs()
+}
+
+
+# current active dataset
+def get_dataset_variables(dataset_name):
+ if dataset_name == "femicides/rai":
+ spacy_model = "it_core_news_md"
+ elif dataset_name == "femicides/olv":
+ spacy_model = "it_core_news_md"
+ elif dataset_name == "crashes/thecrashes":
+ spacy_model = "nl_core_news_md"
+ elif dataset_name == "migration/pavia":
+ spacy_model = "it_core_news_md"
+ else:
+ raise ValueError("Unsupported dataset!")
+
+ return {
+ "dataset": dataset_name,
+ "frames": read_frames_of_interest(dataset_name),
+ "spacy_model": spacy_model,
+ }
+
+
+# ==== DATA PROCESSING FUNCTIONS ====
+# event data
+def load_event_data(dataset):
+ if dataset == "femicides/rai":
+ event_data_file = "output/femicides/split_data/rai/split_ALL.events.csv"
+ texts_data_file = "output/femicides/split_data/rai/split_ALL.texts.meta.csv"
+ elif dataset == "femicides/olv":
+ event_data_file = "output/femicides/split_data/olv/split_dev10.events.csv"
+ texts_data_file = "output/femicides/split_data/olv/split_dev10.texts.csv"
+ elif dataset == "crashes/thecrashes":
+ event_data_file = "output/crashes/split_data/split_dev10.events.csv"
+ texts_data_file = "output/crashes/split_data/split_dev10.texts.meta.csv"
+ elif dataset == "migration/pavia":
+ event_data_file = "output/migration/split_data/split_dev10.events.csv"
+ texts_data_file = "output/migration/split_data/split_dev10.texts.meta.csv"
+ else:
+ raise ValueError("Unsupported dataset")
+ events = pd.read_csv(event_data_file, dtype={"event:id": int}, index_col=0)
+ texts = enrich_texts_df(pd.read_csv(texts_data_file, index_col=0), events)
+ return {"events_df": events, "texts_df": texts}
+
+
+DATASETS = {
+ "femicides/rai": load_event_data("femicides/rai"),
+ "femicides/olv": load_event_data("femicides/olv"),
+ "crashes/thecrashes": load_event_data("crashes/thecrashes"),
+ "migration/pavia": load_event_data("migration/pavia"),
+}
+
+SKIP_FUNCTIONS = {
+ "femicides/rai": None,
+ "femicides/olv": None,
+ "crashes/thecrashes": lambda doc: not is_a_dutch_text(doc),
+ "migration/pavia": None
+}
+
+
+
+def read_frames_to_event_roles(dataset):
+ if dataset == "femicides/rai":
+ ftr_df = pd.read_csv("resources/femicides_frame_to_roles.csv")
+ if dataset == "femicides/olv":
+ ftr_df = pd.read_csv("resources/femicides_frame_to_roles.csv")
+ elif dataset == "crashes/thecrashes":
+ ftr_df = pd.read_csv("resources/crashes_frame_to_roles.csv")
+ else:
+ raise ValueError("Unsupported dataset!")
+
+ frames_to_event_roles: Dict[str, Dict[str, List[str]]] = {}
+ role_types = [col for col in ftr_df.columns if col.startswith("role:")]
+ for _, row in ftr_df.iterrows():
+ frame_roles = defaultdict(list)
+ for rt in role_types:
+ role_key = rt.split(":")[1]
+ if row[rt] == "-":
+ frame_roles[role_key] = []
+ else:
+ for role in row[rt].split("|"):
+ frame_roles[role_key].append(role)
+ frames_to_event_roles[row["frame"]] = frame_roles
+ return frames_to_event_roles
+
+
+def get_role_expressions(
+ struct: FrameStructure,
+ roles_dep_map: Dict[int, Dict[str, str]],
+ frame_to_role_map: Optional[Dict[str, Dict[str, List[str]]]],
+ depth_filter: int,
+ output_depth_only: bool = False,
+) -> str:
+
+ role_exps = []
+ role_deps = roles_dep_map[struct.target.tokens_idx[0]]
+
+ def make_exp(_role, _dep, _depth):
+ if output_depth_only:
+ return _role + "::" + str(_depth)
+ else:
+ if _depth > depth_filter:
+ _dep = None
+ return _role + "::" + (_dep or "_UNK_DEP")
+
+ # no event role mapping: just use roles as-is
+ if frame_to_role_map is None:
+ for role, _ in struct.roles:
+ dep, depth = role_deps.get(role, (None, -1))
+ role_exps.append(make_exp(role, dep, depth))
+ elif struct.frame in frame_to_role_map:
+ for role_type, rt_roles in frame_to_role_map[struct.frame].items():
+ for role in rt_roles:
+ if role in [r[0] for r in struct.roles]:
+ dep, depth = role_deps.get(role, (None, -1))
+ role_exps.append(make_exp(role, dep, depth))
+ # else:
+ # exp = role_type + "::_ABSENT"
+ # role_exps.append(exp)
+ return role_exps
+
+
+def get_analyze_frame_samples(
+ construction,
+ frame,
+ dependency,
+ role,
+ lome_model,
+ max_samples_per_doc,
+ samples_to_find,
+ selected_documents,
+ dataset_vars,
+ texts_df,
+):
+ dataset = dataset_vars["dataset"]
+
+ print("# selected documents", len(selected_documents))
+
+ samples_found = {}
+
+ tar_blocks = get_tarball_blocks(dataset, lome_model)
+
+ syntax_cache = SYNTAX_ANALYSIS_CACHE_FILES[dataset]
+
+ prediction_files = []
+ print("# tar blocks", len(glob.glob(tar_blocks + "/*.tar")))
+ for block in glob.glob(tar_blocks + "/*.tar"):
+ with tarfile.open(block, "r") as tar_f:
+ block_prediction_files = [
+ m.name for m in tar_f.getmembers() if m.name.endswith(".comm.json")
+ ]
+ print("\t# prediction files", len(prediction_files))
+ matching_prediction_files = [
+ pf
+ for pf in block_prediction_files
+ if re.search(r"/(\d+)/lome_(\d+).comm.json", pf).group(2)
+ in selected_documents
+ ]
+ print("\t# matching prediction files", len(matching_prediction_files))
+ print("\t")
+ prediction_files.extend(matching_prediction_files)
+ print(len(prediction_files))
+
+ while prediction_files and len(samples_found) < samples_to_find:
+
+ print(
+ f"\t\tsamples_found: {len(samples_found)}//prediction_files left: {len(prediction_files)}"
+ )
+
+ # choose a random prediction file
+ pf = random.choice(prediction_files)
+ prediction_files.remove(pf)
+
+ print(pf)
+
+ # filter for selected frame
+ doc_id = os.path.basename(pf).split(".")[0].split("_")[1]
+ doc_key = doc_id[:2]
+ tarball = get_tarball_blocks(dataset, lome_model) + f"/block_{doc_key}.tar"
+ with tarfile.open(tarball, "r") as tar_f:
+ pf_obj = io.TextIOWrapper(tar_f.extractfile(pf))
+ (
+ sents,
+ pred_structures,
+ syntax_analyses,
+ role_analyses,
+ ) = process_prediction_file(
+ filename=pf,
+ file_obj=pf_obj,
+ dataset_name=dataset_vars["dataset"],
+ syntax_cache=syntax_cache,
+ deep_frames_cache=deep_frames_cache,
+ )
+
+ if syntax_analyses is None:
+ continue
+
+ (
+ frame_sents,
+ frame_pred_structures,
+ frame_syntax_analyses,
+ frame_role_mappings,
+ ) = ([], [], [], [])
+ for s, pred, syn, rol in zip(
+ sents, pred_structures, syntax_analyses, role_analyses
+ ):
+ for fs in pred.values():
+ fs_syn = get_syntax_info(fs, syn)
+ fs_rol = rol[fs.target.tokens_idx[0]]
+
+ frame_matches = frame == "*" or fs.frame == frame
+ construction_matches = (
+ construction == "*"
+ or fs_syn["syn_construction"] == construction
+ )
+ role_matches = role == "*" or role in [r for r, _ in fs.roles]
+ if role != "*":
+ dependency_matches = dependency == "*" or (
+ role,
+ dependency,
+ ) in [(r, d) for r, (d, _) in fs_rol.items()]
+ else:
+ dependency_matches = dependency == "*" or dependency in [
+ d for d, _ in fs_rol.values()
+ ]
+ if (
+ frame_matches
+ and construction_matches
+ and role_matches
+ and dependency_matches
+ ):
+ frame_sents.append(s)
+ frame_pred_structures.append(pred)
+ frame_syntax_analyses.append(syn)
+ frame_role_mappings.append(rol)
+
+ # select random frames
+ if not frame_sents:
+ continue
+ for _ in range(max_samples_per_doc):
+ selected_idx = random.randrange(len(frame_sents))
+ if not (pf, selected_idx) in samples_found:
+ sample = (
+ frame_sents[selected_idx],
+ frame_pred_structures[selected_idx],
+ frame_syntax_analyses[selected_idx],
+ frame_role_mappings[selected_idx],
+ )
+ if sample not in samples_found.values():
+ samples_found[(pf, selected_idx)] = sample
+
+ # format output
+ output = []
+ for (pf, idx), (sent, structs, syntax, roles) in samples_found.items():
+ # extract eventID and docID from file string
+ re_m = re.search(r"/(\d+)/lome_(\d+).comm.json", pf)
+ event_id = re_m.group(1)
+ doc_id = re_m.group(2)
+
+ output.append(
+ {
+ "sentence": sent,
+ "fn_structures": [
+ dataclasses.asdict(fs) for fs in structs.values()
+ ],
+ "syntax": syntax,
+ "roles": roles,
+ "meta": {
+ "event_id": event_id,
+ "doc_id": doc_id,
+ "text_meta": get_text_meta(doc_id, texts_df),
+ },
+ }
+ )
+ return output
+
+
+# ==== WEB SERVER UTILITY FUNCTIONS ====
+def security_check():
+ return True
+ # if request.cookies.get("auth_key", None) == AUTH_KEY:
+ # return True
+ # if request.args.get("auth_key", None) == AUTH_KEY:
+ # return True
+ # return False
+
+
+# ==== APP ROUTE FUNCTIONS ====
+@app.before_request
+def init_session():
+ if session.get("initialized"):
+ return
+
+ print("Initializing...")
+ _switch_dataset("femicides/olv", True)
+ session["initialized"] = True
+ return
+
+
+@app.route("/")
+def index():
+ return redirect(url_for("demo"))
+
+
+@app.route("/explore")
+def start():
+ return render_template("index.html")
+
+
+@app.route("/demo")
+def demo():
+ return render_template("demo.html")
+
+
+@app.route("/check_password", methods=["POST"])
+def check_password():
+ entered_password = request.form["password"]
+ if entered_password == PASSWORD:
+ resp = jsonify({"success": True})
+ resp.set_cookie("auth_key", AUTH_KEY)
+ return resp
+ else:
+ return jsonify({"success": False})
+
+
+@app.route("/switch_dataset")
+def switch_dataset():
+ new_dataset = request.args.get("dataset")
+ _switch_dataset(new_dataset)
+ return jsonify({"result": "changed_dataset"})
+
+
+def _switch_dataset(new_dataset, first_time=False):
+ print(first_time)
+ if not first_time:
+ if new_dataset == session["dataset_vars"]["dataset"]:
+ return jsonify({"result": "no_change"})
+
+ session["dataset_vars"] = get_dataset_variables(new_dataset)
+ if new_dataset == "femicides/rai":
+ session["provider_name_map"] = {}
+ elif new_dataset == "crashes/thecrashes":
+ _, name_map = read_crashes_provider_attrs()
+ session["provider_name_map"] = name_map
+ elif new_dataset == "migration/pavia":
+ session["provider_name_map"] = {}
+ else:
+ session["provider_name_map"] = {}
+ # session["frames_to_roles"] = read_frames_to_event_roles(new_dataset)
+
+
+@app.route("/analyze")
+def analyze():
+
+ event_id = request.args.get("event")
+ doc_id = request.args.get("document")
+ lome_model = request.args.get("model")
+ dataset = session["dataset_vars"]["dataset"]
+
+ if dataset in PROTECTED_DATASETS:
+ if not security_check():
+ abort(403)
+
+ print(dataset)
+ output = analyze_single_document(
+ doc_id,
+ event_id,
+ lome_model,
+ dataset,
+ DATASETS[session["dataset_vars"]["dataset"]]["texts_df"],
+ deep_frames_cache=deep_frames_cache
+ )
+ return jsonify(output)
+
+
+@app.route("/sample_frame")
+def sample_frame():
+
+ dataset = session["dataset_vars"]["dataset"]
+ if dataset in PROTECTED_DATASETS and not security_check():
+ abort(403)
+
+ frame = request.args.get("frame")
+ construction = request.args.get("construction")
+ role = request.args.get("role")
+ dependency = request.args.get("dependency")
+
+ texts_df = DATASETS[session["dataset_vars"]["dataset"]]["texts_df"]
+ events_df = DATASETS[session["dataset_vars"]["dataset"]]["events_df"]
+ filtered_docs = filter_documents(
+ session["dataset_vars"],
+ events_df,
+ texts_df,
+ PROVIDER_ATTRS[session["dataset_vars"]["dataset"]],
+ session["provider_name_map"],
+ )
+ selected_documents = {doc["doc_id"] for doc in filtered_docs}
+ lome_model = request.args.get("model")
+ samples_to_find = int(request.args.get("nsamples", 5))
+ max_samples_per_doc = int(request.args.get("nperdoc", 10))
+
+ output = get_analyze_frame_samples(
+ construction,
+ frame,
+ dependency,
+ role,
+ lome_model,
+ max_samples_per_doc,
+ samples_to_find,
+ selected_documents,
+ session["dataset_vars"],
+ DATASETS[session["dataset_vars"]["dataset"]]["texts_df"],
+ )
+ return jsonify(output)
+
+
+@app.route("/lus_to_frames")
+def get_frames_from_lus():
+ lus = request.args.get("lus", "").split("+")
+ frames = set()
+ for lu in lus:
+ frames.update({lu_info.frame.name for lu_info in fn.lus(r"^" + lu + r"\.")})
+ print(frames)
+ return jsonify({"frames": sorted(frames)})
+
+
+def format_frame_description(frame_def_xml):
+ frame_def_fmt = [frame_def_xml.text] if frame_def_xml.text else []
+ for elem in frame_def_xml:
+ if elem.tag == "ex":
+ break
+ elif elem.tag == "fen":
+ frame_def_fmt.append("
" + elem.text + "")
+ elif elem.text:
+ frame_def_fmt.append(elem.text)
+ if elem.tail:
+ frame_def_fmt.append(elem.tail)
+ return frame_def_fmt
+
+
+def get_alt_perspectives(frame_info, frame_name):
+ alt_perspectives = []
+ result_frames = [fr.subFrameName for fr in frame_info.frameRelations if fr.type.name == "Causative_of" and fr.superFrameName == frame_name]
+ if result_frames:
+ alt_perspectives.append({"frame": result_frames[0], "type": "result"})
+
+ cause_frames = [fr.superFrameName for fr in frame_info.frameRelations if fr.type.name == "Causative_of" and fr.subFrameName == frame_name]
+ if cause_frames:
+ alt_perspectives.append({"frame": cause_frames[0], "type": "causer"})
+
+ # user_frames = [fr.subFrameName for fr in frame_info.frameRelations if fr.type.name == "Using" and fr.superFrameName == frame_name]
+ # if user_frames:
+ # alt_perspectives.append({"frame": user_frames[0], "type": "user"})
+
+ neutral_frames = [fr.superFrameName for fr in frame_info.frameRelations if fr.type.name == "Perspective_on" and fr.subFrameName == frame_name]
+ if neutral_frames:
+ flipped_frames = [fr.subFrameName for fr in fn.frame(neutral_frames[0]).frameRelations if fr.type.name == "Perspective_on" and fr.superFrameName == neutral_frames[0] and fr.subFrameName != frame_name]
+ if flipped_frames:
+ alt_perspectives.extend([{"frame": ff, "type": "flipped"} for ff in flipped_frames])
+
+ return alt_perspectives
+
+
+@app.route("/frame_info")
+def get_frame_info():
+ frame_name = request.args.get("frame").strip()
+ try:
+ print(repr(frame_name))
+ print(type(frame_name))
+ try:
+ frame_info = fn.frame(frame_name)
+ except KeyError:
+ # sometimes NLTK (randomly??) throws a key error, in which case, just try again
+ frame_info = fn.frame(frame_name)
+ try:
+ # try extracting just the first sentence
+ definition_first_sent = nltk.sent_tokenize(frame_info.definitionMarkup)[0] + ""
+ frame_def_xml = ET.fromstring(definition_first_sent)
+ except ET.XMLSyntaxError:
+ # otherwise, use the full definition
+ frame_def_xml = ET.fromstring(frame_info.definitionMarkup)
+
+ frame_def_fmt = format_frame_description(frame_def_xml)
+
+ exemplars = [
+ {
+ "text": exemplar.text,
+ "target_lu": lu_name,
+ "target_idx": list(exemplar["Target"][0]),
+ "core_fes": {
+ role: exemplar.text[start_idx:end_idx]
+ for role, start_idx, end_idx in exemplar.FE[0]
+ if role in [fe for fe, fe_info in frame_info.FE.items() if fe_info.coreType == "Core"]
+ }
+ }
+ for lu_name, lu_info in frame_info["lexUnit"].items()
+ for exemplar in lu_info.exemplars
+ ]
+
+ # try to find a "typical" exemplar --- typical -> as short as possible, as many FEs as possible
+ exa_typicality_scores = [(exa, len(exa["text"]) - 25 * len(exa["core_fes"])) for exa in exemplars]
+ if exa_typicality_scores:
+ typical_exemplar = min(exa_typicality_scores, key=lambda t: t[1])[0]
+ else:
+ typical_exemplar = None
+
+ alt_perspectives = get_alt_perspectives(frame_info, frame_name)
+
+ return jsonify({
+ "result": "OK",
+ "frameDefinition": frame_def_fmt,
+ "exemplars": exemplars,
+ "altPerspectives": alt_perspectives,
+ "typicalExemplar": {
+ "text": typical_exemplar["text"],
+ "coreFrameElements": typical_exemplar["core_fes"]
+ } if typical_exemplar else None
+ })
+
+ except FramenetError:
+ return jsonify({"result": "FAIL", "info": "could not find frame"})
+
+
+
+@app.route("/frames")
+def get_frames():
+ return jsonify(session["dataset_vars"]["frames"])
+
+
+@app.route("/constructions")
+def get_constructions():
+ return jsonify(POSSIBLE_CONSTRUCTIONS)
+
+
+@app.route("/event_filters")
+def get_event_filters():
+ dataset = session["dataset_vars"]["dataset"]
+ if dataset in PROTECTED_DATASETS and not security_check():
+ abort(403)
+
+ if session["dataset_vars"]["dataset"] == "femicides/rai":
+ events_df = DATASETS[session["dataset_vars"]["dataset"]]["events_df"]
+ event_categories = events_df["event:category"].unique().tolist()
+ regions = sorted(events_df["event:region"].unique().tolist())
+ sem_location = sorted(events_df["event:semantic_location"].unique().tolist())
+ victim_age = VICTIM_AGE_GROUPS
+ victim_nationality = [
+ "Italia",
+ ALL_FOREIGN_NATIONALITIES,
+ "non rilevato",
+ ] + sorted(
+ [
+ i
+ for i in events_df["victim:nationality"].dropna().unique().tolist()
+ if i not in ["Italia", "non rilevato"]
+ ]
+ )
+ attacker_nationality = [
+ "Italia",
+ ALL_FOREIGN_NATIONALITIES,
+ "non rilevato",
+ ] + sorted(
+ [
+ i
+ for i in events_df["attacker:nationality"].dropna().unique().tolist()
+ if i not in ["Italia", "non rilevato"]
+ ]
+ )
+
+ victim_occupation = sorted(
+ [
+ i
+ for i in events_df["victim:occupation"].dropna().unique().tolist()
+ ]
+ )
+
+ attacker_occupation = sorted(
+ [
+ i
+ for i in events_df["attacker:occupation"].dropna().unique().tolist()
+ ]
+ )
+
+
+ return jsonify(
+ {
+ "event_categories": event_categories,
+ "regions": regions,
+ "sem_location": sem_location,
+ "victim_age": victim_age,
+ "victim_nationality": victim_nationality,
+ "attacker_nationality": attacker_nationality,
+ "victim_occupation": victim_occupation,
+ "attacker_occupation": attacker_occupation
+ }
+ )
+
+ elif session["dataset_vars"]["dataset"] == "femicides/olv":
+ events_df = DATASETS[session["dataset_vars"]["dataset"]]["events_df"]
+ regions = sorted([str(r) for r in events_df["event:region"].unique().tolist()])
+ victim_age = VICTIM_AGE_GROUPS
+
+ return jsonify(
+ {
+ "regions": regions,
+ "victim_age": victim_age,
+ }
+ )
+
+
+ elif session["dataset_vars"]["dataset"] == "crashes/thecrashes":
+ events_df = DATASETS[session["dataset_vars"]["dataset"]]["events_df"]
+ filters = {
+ "outcomes": ["no one", "one or more people"],
+ "imbalanced": ["yes", "no"],
+ }
+ return jsonify(filters)
+
+ else:
+ return jsonify({})
+
+
+@app.route("/dep_labels")
+def get_dep_labels():
+ dep_labels = set()
+ with open("resources/dep_labels.txt", encoding="utf-8") as f:
+ for line in f:
+ dep_labels.add(line.strip())
+ return jsonify(sorted(dep_labels))
+
+
+@app.route("/role_labels")
+def get_role_labels():
+ frame = request.args.get("frame")
+ roles = fn_frames_to_roles.get(frame)
+ if roles is not None:
+ return jsonify(roles)
+ else:
+ return jsonify([])
+
+
+@app.route("/doc_filters")
+def get_doc_filters():
+
+ texts_df = DATASETS[session["dataset_vars"]["dataset"]]["texts_df"]
+
+ provider_attrs = PROVIDER_ATTRS[session["dataset_vars"]["dataset"]]
+
+ if session["dataset_vars"]["dataset"] == "crashes/thecrashes":
+
+ name_map = session["provider_name_map"]
+ providers = sorted(
+ texts_df["provider"]
+ .apply(lambda prov: name_map.get(prov))
+ .dropna()
+ .unique()
+ .tolist()
+ )
+
+ provider_provinces = sorted(
+ key.split(":")[1]
+ for key in provider_attrs
+ if key.startswith("Province:")
+ )
+ provider_content_types = sorted(
+ key.split(":")[1]
+ for key in provider_attrs
+ if key.startswith("ContentType:")
+ )
+
+ provider_medium_types = sorted(
+ key.split(":")[1]
+ for key in provider_attrs
+ if key.startswith("MediumType:")
+ )
+
+ provider_owners = sorted(
+ key.split(":")[1]
+ for key in provider_attrs
+ if key.startswith("MediaOwner:")
+ )
+
+ else:
+ providers = sorted(texts_df["provider"].dropna().unique().tolist())
+ provider_provinces = []
+ provider_content_types = []
+ provider_medium_types = []
+ provider_owners = []
+
+ return jsonify(
+ {
+ "providers": providers,
+ "provider_provinces": provider_provinces,
+ "provider_content_types": provider_content_types,
+ "provider_medium_types": provider_medium_types,
+ "provider_owners": provider_owners,
+ }
+ )
+
+
+def apply_doc_filters(
+ doc_filters: List[str],
+ provider_attrs: dict,
+ prov_name_map: dict,
+ texts_df: pd.DataFrame,
+):
+ if not doc_filters:
+ all_doc_ids = set(int(eid) for eid in texts_df["text_id"].tolist())
+ return all_doc_ids
+
+
+ filters_attr_values = defaultdict(list)
+ for doc_filter in doc_filters:
+ _, attribute, value = doc_filter.split("::")
+ filters_attr_values[attribute].append(value)
+
+ selected_docs = texts_df
+ for attribute, values in filters_attr_values.items():
+ attr_conditions = []
+
+ for value in values:
+ if attribute == "days_after":
+
+ # if a filter for `days_after` is set, first remove any rows with null values
+ selected_docs = selected_docs.dropna(subset=["days_after_event"])
+
+ if value == "day":
+ condition = selected_docs["days_after_event"] < 1
+ elif value == "week":
+ condition = selected_docs["days_after_event"].isin(range(1, 7))
+ elif value == "month":
+ condition = selected_docs["days_after_event"].isin(range(7, 30))
+ elif value == "year":
+ condition = selected_docs["days_after_event"].isin(range(30, 365))
+ else:
+ condition = selected_docs["days_after_event"] > 365
+
+ elif session["dataset_vars"]["dataset"] == "femicides/rai":
+ if any(attribute.startswith(key) for key in ["politics:", "type", "area"]):
+ providers = provider_attrs[attribute + ":" + value]
+ condition = selected_docs["provider"].isin(providers)
+ else:
+ condition = selected_docs[attribute] == value
+
+ elif session["dataset_vars"]["dataset"] == "femicides/olv":
+ condition = selected_docs[attribute] == value
+
+ elif session["dataset_vars"]["dataset"] == "crashes/thecrashes":
+
+ if attribute == "provider":
+ condition = selected_docs["provider"].apply(lambda prov: prov_name_map.get(prov)) == value
+
+ elif attribute in [
+ "area",
+ "country",
+ "province",
+ "content_type",
+ "medium_type",
+ "owner",
+ ]:
+ # map to the correct name in the spreadsheet
+ attribute_altname = {
+ "area": "RegionalScope",
+ "country": "Country",
+ "province": "Province",
+ "content_type": "ContentType",
+ "medium_type": "MediumType",
+ "owner": "MediaOwner",
+ }[attribute]
+
+ providers = provider_attrs[attribute_altname + ":" + value]
+ condition = selected_docs["provider"].apply(lambda prov: prov_name_map.get(prov)).isin(providers)
+ else:
+ condition = selected_docs[attribute] == value
+
+ elif session["dataset_vars"]["dataset"] == "migration/pavia":
+ if attribute.startswith("politics") or attribute.startswith("religion"):
+ providers = provider_attrs[attribute + ":" + value]
+ condition = selected_docs["provider"].isin(providers)
+ else:
+ condition = selected_docs[attribute] == value
+
+ attr_conditions.append(condition)
+
+ selected_docs = selected_docs[functools.reduce(np.logical_or, attr_conditions)]
+
+ return set(int(eid) for eid in selected_docs["text_id"].tolist())
+
+
+def apply_event_filters(ev_filters: List[str], events_df: pd.DataFrame):
+ if not ev_filters:
+ all_event_ids = set(int(eid) for eid in events_df["event:id"].tolist())
+ return all_event_ids
+
+ selected_events = events_df
+ for ev_filter in ev_filters:
+ print(ev_filter)
+ _, attribute, value = ev_filter.split("::")
+ print(attribute)
+
+ if session["dataset_vars"]["dataset"] in ["femicides/rai", "femicides/olv"]:
+ if attribute in ["victim:age"]:
+ print(value)
+ if "-" not in value:
+ # age_from = int(value.split("+")[0])
+ age_from = int(value)
+ age_to = 200
+ else:
+ age_from = int(value.split("-")[0])
+ age_to = int(value.split("-")[1])
+ events_with_age = selected_events[
+ selected_events[attribute] != "non rilevato"
+ ]
+ selected_events = events_with_age[
+ events_with_age[attribute].astype(int).isin(range(age_from, age_to))
+ ]
+ elif attribute in ["victim:nationality", "attacker:nationality"]:
+ if value == ALL_FOREIGN_NATIONALITIES:
+ selected_events = selected_events.dropna(subset=[attribute])
+ selected_events = selected_events[
+ ~selected_events[attribute].isin(["Italia", "non rilevato", "nessuno", "sconosciuto"])
+ ]
+ else:
+ selected_events = selected_events[
+ selected_events[attribute] == value
+ ]
+ else:
+ selected_events = selected_events[selected_events[attribute] == value]
+
+ elif session["dataset_vars"]["dataset"] == "crashes/thecrashes":
+
+ if attribute.startswith("imbalanced"):
+ # at least one pedestrian or cyclist was injured or died
+ selected_events = selected_events[
+ (selected_events["outcomes:dead:cyclist"] > 0)
+ | (selected_events["outcomes:dead:pedestrian"] > 0)
+ | (selected_events["outcomes:injured:cyclist"] > 0)
+ | (selected_events["outcomes:injured:pedestrian"] > 0)
+ ]
+
+ # no person in a vehicle was injured or died
+ selected_events = selected_events[
+ (selected_events["outcomes:injured:vehicle"] == 0)
+ & (selected_events["outcomes:dead:vehicle"] == 0)
+ ]
+
+ # vehicle was involved
+ selected_events = selected_events[
+ (selected_events["vehicle_involved"] == 1)
+ ]
+
+ if attribute.startswith("outcomes:"):
+
+ outcome = attribute.split(":")[1]
+ person = attribute.split(":")[2]
+
+ if outcome == "deadinjured":
+ if person == "cyclistpedestrian":
+ if value == "no one":
+ selected_events = selected_events[
+ (selected_events["outcomes:dead:cyclist"] == 0)
+ & (selected_events["outcomes:dead:pedestrian"] == 0)
+ & (selected_events["outcomes:injured:cyclist"] == 0)
+ & (selected_events["outcomes:injured:pedestrian"] == 0)
+ ]
+ else:
+ selected_events = selected_events[
+ (selected_events["outcomes:dead:cyclist"] > 0)
+ | (selected_events["outcomes:dead:pedestrian"] > 0)
+ | (selected_events["outcomes:injured:cyclist"] > 0)
+ | (selected_events["outcomes:injured:pedestrian"] > 0)
+ ]
+ else:
+ if value == "no one":
+ selected_events = selected_events[
+ (selected_events[f"outcomes:dead:{person}"] == 0)
+ & (selected_events[f"outcomes:injured:{person}"] == 0)
+ ]
+ else:
+ selected_events = selected_events[
+ (selected_events[f"outcomes:dead:{person}"] > 0)
+ | (selected_events[f"outcomes:injured:{person}"] > 0)
+ ]
+
+ else:
+ if person == "cyclistpedestrian":
+ if value == "no one":
+ selected_events = selected_events[
+ (selected_events[f"outcomes:{outcome}:cyclist"] == 0)
+ & (
+ selected_events[f"outcomes:{outcome}:pedestrian"]
+ == 0
+ )
+ ]
+ else:
+ selected_events = selected_events[
+ (selected_events[f"outcomes:{outcome}:cyclist"] == 0)
+ | (
+ selected_events[f"outcomes:{outcome}:pedestrian"]
+ > 0
+ )
+ ]
+ else:
+ if value == "no one":
+ selected_events = selected_events[
+ selected_events[attribute] == 0
+ ]
+ else:
+ selected_events = selected_events[
+ selected_events[attribute] > 0
+ ]
+
+ return set(int(eid) for eid in selected_events["event:id"].tolist())
+
+
+@app.route("/documents")
+def documents():
+
+ events_df = DATASETS[session["dataset_vars"]["dataset"]]["events_df"]
+ texts_df = DATASETS[session["dataset_vars"]["dataset"]]["texts_df"]
+ document_list = filter_documents(
+ session["dataset_vars"],
+ events_df,
+ texts_df,
+ PROVIDER_ATTRS[session["dataset_vars"]["dataset"]],
+ session["provider_name_map"],
+ )
+ return jsonify(sorted(document_list, key=lambda d: int(d["event_id"])))
+
+
+def filter_documents(dataset_vars, events_df, texts_df, provider_attrs, name_map):
+ event_filters = read_filters("event_filters")
+ doc_filters = read_filters("doc_filters")
+ selected_events = apply_event_filters(event_filters, events_df)
+ selected_documents = apply_doc_filters(
+ doc_filters, provider_attrs, name_map, texts_df
+ )
+
+ document_list = []
+
+ blocks = get_tarball_blocks(dataset_vars["dataset"])
+ for tarball in glob.glob(blocks + "/*.tar"):
+ with tarfile.open(tarball, "r") as tar_f:
+ for doc in [f.name for f in tar_f.getmembers() if f.name.endswith(".comm.txt")]:
+ # extract eventID and docID from file string
+ re_m = re.search(r"/(\d+)/lome_(\d+).comm.txt", doc)
+ event_id = re_m.group(1)
+ doc_id = re_m.group(2)
+ if (int(event_id) not in selected_events) or (
+ int(doc_id) not in selected_documents
+ ):
+ continue
+ document_list.append({"event_id": event_id, "doc_id": doc_id})
+ return document_list
+
+
+def read_filters(arg_name):
+ filter_str = request.args.get(arg_name)
+ if filter_str:
+ filters = filter_str.split("+")
+ else:
+ filters = []
+ return filters
+
+
+@app.route("/frame_freq")
+def frame_freq():
+
+ texts_df = DATASETS[session["dataset_vars"]["dataset"]]["texts_df"]
+
+ # arg: "model"
+ lome_model = request.args.get("model")
+
+ # arg: "filter"
+ event_filters = read_filters("event_filters")
+ doc_filters = read_filters("doc_filters")
+ events_df = DATASETS[session["dataset_vars"]["dataset"]]["events_df"]
+ selected_events = apply_event_filters(event_filters, events_df)
+ selected_documents = apply_doc_filters(
+ doc_filters,
+ PROVIDER_ATTRS[session["dataset_vars"]["dataset"]],
+ session["provider_name_map"],
+ texts_df,
+ )
+
+ # arg: "headlines" (consider only headlines?)
+ only_headlines = request.args.get("headlines", "n") == "y"
+
+ # arg: "frames"
+ frame_string = request.args.get("frames").strip()
+ frame_filter: List[str] = frame_string.split("+")
+
+ # arg: "constructions"
+ constr_string = request.args.get("constructions").strip()
+ constr_filter: List[str] = constr_string.split("+") if constr_string else []
+
+ # arg: "group_by_cat"
+ group_by_cat: bool = request.args.get("group_by_cat") == "y"
+
+ # arg: "group_by_tgt"
+ group_by_tgt: bool = request.args.get("group_by_tgt") == "y"
+
+ # arg: "group_by_constr"
+ group_by_constr: bool = request.args.get("group_by_constr") == "y"
+
+ # arg: "group_by_root"
+ group_by_root: bool = request.args.get("group_by_root") == "y"
+
+ # arg: "group_by_role_expression"
+ group_by_role_expr: int = int(request.args.get("group_by_role_expr"))
+
+ # arg: "relative"
+ relative: bool = request.args.get("relative") == "y"
+
+ # arg: "plot_over_days_post"
+ plot_over_days_post: bool = request.args.get("plot_over_days_post") == "y"
+
+ #arg: "plot_by_year"
+ plot_by_year: bool = request.args.get("plot_by_year") == "y"
+ assert not (plot_over_days_post and plot_by_year)
+
+ # arg: "days_time_window"
+ days_time_window: int
+ try:
+ days_time_window_str = request.args.get("days_time_window")
+ if days_time_window_str is None:
+ days_time_window = 10
+ else:
+ days_time_window = int(days_time_window_str)
+ except ValueError:
+ days_time_window = 10
+
+ if plot_over_days_post or plot_by_year:
+ relevant_frame_counts = defaultdict(lambda: defaultdict(int))
+ deep_frame_counts = defaultdict(lambda: defaultdict(int))
+ all_frame_counts = defaultdict(lambda: defaultdict(int))
+ else:
+ relevant_frame_counts = defaultdict(int)
+ deep_frame_counts = defaultdict(int)
+ all_frame_counts = defaultdict(int)
+
+ # total times each frame is found
+ totals_by_frame = defaultdict(int)
+
+ print("Processing documents....")
+
+ blocks = get_tarball_blocks(session["dataset_vars"]["dataset"], lome_model)
+
+ # allow share syntax cache across different calls of process_prediction_file()
+ tmp_syntax_cache = {}
+
+ for tarball in sorted(glob.glob(blocks + "/*.tar")):
+ with tarfile.open(tarball, "r") as tar_f:
+ for mem in sorted(tar_f.getmembers(), key=lambda mem: mem.name):
+ if mem is None or not mem.name.endswith(".comm.json"):
+ continue
+
+ # extract eventID and docID from file string
+ re_m = re.search(r"/(\d+)/lome_(\d+).comm.json", mem.name)
+ event_id = re_m.group(1)
+ doc_id = re_m.group(2)
+
+ # document / event filter
+ if (int(doc_id) not in selected_documents) or (
+ int(event_id) not in selected_events
+ ):
+ continue
+ if plot_over_days_post:
+ texts_df = DATASETS[session["dataset_vars"]["dataset"]]["texts_df"]
+ texts_df_dropna = texts_df.dropna(subset=["days_after_event"])
+ try:
+ df_filter = texts_df_dropna["text_id"] == int(doc_id)
+ time_bucket = int(
+ texts_df_dropna[df_filter].iloc[0]["days_after_event"]
+ )
+ except IndexError:
+ # ignore files with null days_post values
+ continue
+ # adjust negative values
+ time_bucket = max(time_bucket, 0)
+
+ # round to 10-day periods
+ time_bucket = (time_bucket // days_time_window) * days_time_window
+ else:
+ time_bucket = 0
+
+ if plot_by_year:
+ texts_df = DATASETS[session["dataset_vars"]["dataset"]]["texts_df"]
+ df_filter = texts_df["text_id"] == int(doc_id)
+ if "pubyear" in texts_df.columns:
+ time_bucket = int(texts_df[df_filter].iloc[0]["pubyear"])
+ elif "pubdate" in texts_df.columns:
+ pubdate_str = texts_df[df_filter].iloc[0]["pubdate"]
+ if pd.isna(pubdate_str):
+ continue
+ pub_date = datetime.datetime.strptime(pubdate_str, "%Y-%m-%d %H:%M:%S")
+ time_bucket = pub_date.year
+ else:
+ raise ValueError("Cannot plot by year if no `pubyear` or `pubdate` is specified!")
+ # continue
+
+ mem_obj = io.TextIOWrapper(tar_f.extractfile(mem))
+
+ skip_func = SKIP_FUNCTIONS[session["dataset_vars"]["dataset"]]
+ if skip_func is not None and skip_func(doc_id):
+ print(f"\tskip_func: skipping file {mem}")
+ continue
+
+ # spacy_model = session["dataset_vars"]["spacy_model"]
+ (
+ _,
+ pred_structures,
+ syntax_analyses,
+ role_analyses,
+ ) = process_prediction_file(
+ filename=mem.name,
+ dataset_name=session["dataset_vars"]["dataset"],
+ syntax_cache=SYNTAX_ANALYSIS_CACHE_FILES[session["dataset_vars"]["dataset"]],
+ tmp_cache=tmp_syntax_cache,
+ file_obj=mem_obj,
+ deep_frames_cache=deep_frames_cache
+ )
+
+ for sent_idx, (struct_dict, syntax_dict, roles) in enumerate(zip(
+ pred_structures, syntax_analyses, role_analyses
+ )):
+ if only_headlines and sent_idx > 1:
+ continue
+ for struct in struct_dict.values():
+
+ frame_key = struct.frame
+ deep_frame_key = struct.deep_frame
+
+ syntax_info = get_syntax_info(struct, syntax_dict)
+ syntax_constr = syntax_info["syn_construction"]
+ syntax_cat = syntax_info["syn_category"]
+ syntax_at_root = is_at_root(syntax_info)
+
+ if constr_filter and syntax_constr not in constr_filter:
+ continue
+
+ totals_by_frame[struct.frame] += 1
+
+ if group_by_cat:
+ count_keys = [f"{frame_key}::{syntax_cat}"]
+ deep_count_keys = [f"{frame_key}::{syntax_cat}"]
+
+ elif group_by_tgt:
+ tgt_str = (
+ " ".join(struct.target.tokens_str)
+ .strip("«».,()□�?'\"")
+ .strip()
+ .lower()
+ )
+ count_keys = [f"{frame_key}::{tgt_str}"]
+ deep_count_keys = [f"{frame_key}::{tgt_str}"]
+
+ elif group_by_constr and group_by_root:
+ count_keys = [
+ f"{frame_key}/{syntax_constr}::{'root' if syntax_at_root else 'non-root'}"
+ ]
+ deep_count_keys = [
+ f"{deep_frame_key}::{syntax_constr}::{'root' if syntax_at_root else 'non-root'}"
+ ]
+ elif group_by_constr:
+ count_keys = [f"{frame_key}::{syntax_constr}"]
+ deep_count_keys = [f"{deep_frame_key}::{syntax_constr}"]
+ elif group_by_root:
+ count_keys = [
+ f"{frame_key}::{'root' if syntax_at_root else 'non-root'}"
+ ]
+ deep_count_keys = [
+ f"{deep_frame_key}::{'root' if syntax_at_root else 'non-root'}"
+ ]
+
+ elif group_by_role_expr:
+ if group_by_role_expr == 1:
+ role_exprs = [r for r, _ in struct.roles]
+ elif group_by_role_expr == 2:
+ role_exprs = get_role_expressions(
+ struct, roles, None, 1, False
+ )
+ elif group_by_role_expr == 3:
+ role_exprs = get_role_expressions(
+ struct, roles, session["frames_to_roles"], 1, False
+ )
+ elif group_by_role_expr == 4:
+ role_exprs = get_role_expressions(
+ struct, roles, None, None, True
+ )
+ else:
+ raise ValueError(
+ "Unknown value for param group_by_role_expr"
+ )
+ count_keys = []
+ deep_count_keys = []
+ for role_expr in role_exprs:
+
+ if group_by_role_expr == 4:
+ role_name, depth = role_expr.split("::")
+ depth = abs(int(depth))
+ if depth > 3:
+ depth = ">3"
+ role_expr = f"{role_name}::{depth}"
+
+ count_keys.append(f"{frame_key}::{role_expr}")
+ deep_count_keys.append(f"{deep_frame_key}::{role_expr}")
+ else:
+ count_keys = [struct.frame]
+ deep_count_keys = [struct.deep_frame]
+
+ for ck, dck in zip(count_keys, deep_count_keys):
+ if struct.frame in frame_filter:
+ if plot_over_days_post or plot_by_year:
+ relevant_frame_counts[time_bucket][ck] += 1
+ deep_frame_counts[time_bucket][dck] += 1
+ else:
+ relevant_frame_counts[ck] += 1
+ deep_frame_counts[dck] += 1
+ if plot_over_days_post or plot_by_year:
+ all_frame_counts[time_bucket][ck] += 1
+ else:
+ all_frame_counts[ck] += 1
+ print("Computing frame counts...")
+
+ if plot_over_days_post or plot_by_year:
+ data_and_names = [
+ (relevant_frame_counts, "relevant_frame_counts"),
+ (deep_frame_counts, "deep_frame_counts"),
+ (all_frame_counts, "all_frame_counts"),
+ ]
+ data_out = {}
+ for (data, name) in data_and_names:
+ traces = defaultdict(lambda: {"x": [], "y": []})
+ for time_bucket in sorted(data):
+ total_count = sum(data[time_bucket].values())
+ for count_keys, count in data[time_bucket].items():
+ traces[count_keys]["x"].append(time_bucket)
+ traces[count_keys]["y"].append(
+ count / total_count if relative else count
+ )
+ data_out[name] = traces
+ return jsonify(data_out)
+
+ else:
+ relevant_frames_sr = pd.Series(data=relevant_frame_counts).sort_values(
+ ascending=False
+ )
+ deep_frames_sr = pd.Series(data=deep_frame_counts).sort_values(ascending=False)
+ all_frames_sr = pd.Series(data=all_frame_counts).sort_values(ascending=False)
+
+ if relative and group_by_role_expr > 0:
+ print("totals_by_frame=", totals_by_frame)
+ print("frame_filter=", frame_filter)
+ denom = totals_by_frame[frame_filter[0]]
+ print("denom=", denom)
+ relevant_frames_sr /= denom
+ deep_frames_sr /= deep_frames_sr.sum() # TODO: what to do with this???
+ all_frames_sr /= all_frames_sr.sum()
+
+ elif relative:
+ relevant_frames_sr /= relevant_frames_sr.sum()
+ deep_frames_sr /= deep_frames_sr.sum()
+ all_frames_sr /= all_frames_sr.sum()
+
+ return jsonify(
+ {
+ "relevant_frame_counts": {
+ "x": relevant_frames_sr.index.tolist(),
+ "y": relevant_frames_sr.values.tolist(),
+ },
+ "deep_frame_counts": {
+ "x": deep_frames_sr.index.tolist(),
+ "y": deep_frames_sr.values.tolist(),
+ },
+ "all_frame_counts": {
+ "x": all_frames_sr.index.tolist(),
+ "y": all_frames_sr.values.tolist(),
+ },
+ }
+ )
+
+
+
+# for demo app
+@app.route("/similar_frames")
+def similar_frames():
+
+ if gensim_m is None:
+ return jsonify({"result": "FAIL", "reason": "no GENSIM model has been loaded, please call /load_gensim and try again"})
+
+ words_in = [w for w in request.args.get("words_in").split("+") if "glove_" + w in gensim_m]
+ if not words_in:
+ return jsonify({"result": "FAIL", "reason": "No input words given"})
+
+ try:
+ matches = [res for res in gensim_m.most_similar(positive=["glove_" + w for w in words_in], topn=100) if res[0].startswith("fn_")]
+ except KeyError:
+ return jsonify({"result": "FAIL", "reason": "One of the input words does not exist in the GloVe vocabulary"})
+
+ frames = [m[0].lstrip("fn_") for m in matches]
+ probas = [m[1] for m in matches]
+
+ return jsonify({
+ "result": "OK",
+ "frames": frames,
+ "probabilities": probas,
+ })
+
+
+@app.route("/sociofillmore")
+def sociofillmore():
+
+ # step 1: LOME analysis
+ input_text = request.args.get("text", "")
+ language = request.args.get("language", "en")
+ sentences = [s.text for s in spacy_model_ud(input_text).sents]
+
+
+ r = requests.get("http://localhost:9090/analyze", {"text": "\n".join(sentences)})
+ # r = requests.get("http://lome:9090/analyze", {"text": "\n".join(sentences)})
+ lome_analyses = json.loads(r.text)["analyses"]
+
+ # intermediate step: make temporary files for saving input/output analyses
+ tmp_in = tempfile.NamedTemporaryFile(mode="w+", delete=False)
+ tmp_in.write(json.dumps(lome_analyses))
+ tmp_in.close()
+
+ tmp_out = tempfile.NamedTemporaryFile(mode="w+", delete=False)
+ tmp_out.close()
+
+ # step 2: SocioFillmore
+ analyze_external_file(tmp_in.name, tmp_out.name, spacy_model_langs[language])
+
+ with open(tmp_out.name, "r") as f_out:
+ data_out = json.load(f_out)
+
+ os.unlink(tmp_in.name)
+ os.unlink(tmp_out.name)
+
+ return jsonify(data_out)
+
+
+if __name__ == "__main__":
+ from waitress import serve
+
+ if len(sys.argv) > 1:
+ host = sys.argv[1]
+ else:
+ host = "127.0.0.1"
+
+ debug = False
+ if len(sys.argv) > 2:
+ if sys.argv[2] == "debug":
+ debug = True
+
+ serve(app, host="0.0.0.0", port="5000")
+ # app.run(host=host, debug=False, ssl_context="adhoc")
diff --git a/sociofillmore/webapp/query_frame_samples.py b/sociofillmore/webapp/query_frame_samples.py
new file mode 100755
index 0000000000000000000000000000000000000000..796b8e19eff695ad3121d7540158f0563bfc8141
--- /dev/null
+++ b/sociofillmore/webapp/query_frame_samples.py
@@ -0,0 +1,117 @@
+import sys
+import requests
+import json
+
+import pandas as pd
+
+
+SOCIOFILLMORE_API = "http://127.0.0.1:5000"
+AUTH_KEY = "3TrJ397oh#^"
+
+
+def get_sample(s, dataset, n_samples, frame, construction, role, dependency):
+
+ s.get(SOCIOFILLMORE_API + "/switch_dataset", params={"dataset": dataset})
+
+ r_q = s.get(
+ SOCIOFILLMORE_API + "/sample_frame",
+ params={
+ "auth_key": AUTH_KEY,
+ "frame": frame,
+ "construction": construction,
+ "role": role,
+ "dependency": dependency,
+ "model": "lome_0shot",
+ "n": n_samples,
+ },
+ )
+
+ data = json.loads(r_q.text)
+
+ rows_out = []
+
+ for sent in data:
+ for fns in sent["fn_structures"]:
+ if fns["frame"] == frame:
+ target_roles = [r for r in fns["roles"] if r[0] == role]
+ if target_roles:
+ target_role = target_roles[0]
+ else:
+ continue
+
+ rows_out.append(
+ {
+ "dataset": dataset,
+ "sentence": " ".join(sent["sentence"]),
+ "frame": frame,
+ "target": " ".join(fns["target"]["tokens_str"]),
+ "role_label": role,
+ "role_span": " ".join(target_role[1]["tokens_str"]),
+ "dependency": dependency,
+ }
+ )
+
+ return rows_out
+
+
+def get_labels(s, dataset, frame):
+ s.get(SOCIOFILLMORE_API + "/switch_dataset", params={"dataset": dataset})
+
+ r_q = s.get(
+ SOCIOFILLMORE_API + "/frame_freq",
+ params={
+ "auth_key": AUTH_KEY,
+ "model": "lome_0shot",
+ "frames": frame,
+ "constructions": "",
+ "group_by_cat": "n",
+ "group_by_constr": "n",
+ "group_by_role_expr": 2,
+ "relative": "y",
+ "plot_over_days_post": "n",
+ },
+ )
+
+ data = json.loads(r_q.text)
+ return {l.split("::")[2] for l in data["relevant_frame_counts"]["x"]}
+
+
+def main(language):
+
+ s = requests.Session()
+
+
+ if language == "it":
+ print("Finding IT labels...")
+ labels_it = get_labels(s, "femicides/rai", "Killing")
+ sample_rows_it = []
+ for label in sorted(labels_it):
+
+ if label == "_UNK_DEP":
+ continue
+
+ print(f"Label (IT): {label}")
+ sample_rows_it.extend(get_sample(s, "femicides/rai", 2, "Killing", "*", "Killer", label))
+ sample_rows_it.extend(get_sample(s, "femicides/rai", 2, "Killing", "*", "Victim", label))
+
+ df_samples_it = pd.DataFrame(sample_rows_it)
+ df_samples_it.to_csv("output/common/query_frame_samples/it_dep_samples.csv")
+
+ if language == "nl":
+ print("Finding NL labels...")
+ labels_nl = get_labels(s, "crashes/thecrashes", "Cause_harm")
+ sample_rows_nl = []
+ for label in sorted(labels_nl):
+
+ if label == "_UNK_DEP":
+ continue
+
+ print(f"Label (NL): {label}")
+ sample_rows_nl.extend(get_sample(s, "crashes/thecrashes", 2, "Cause_harm", "*", "Agent", label))
+ sample_rows_nl.extend(get_sample(s, "crashes/thecrashes", 2, "Cause_harm", "*", "Victim", label))
+ df_samples_nl = pd.DataFrame(sample_rows_nl)
+ df_samples_nl.to_csv("output/common/query_frame_samples/nl_dep_samples.csv")
+
+
+if __name__ == "__main__":
+ main(language=sys.argv[1])
diff --git a/sociofillmore/webapp/static/app.js b/sociofillmore/webapp/static/app.js
new file mode 100755
index 0000000000000000000000000000000000000000..1fd2fcb5fab618c3ebf24c546f65923730f41602
--- /dev/null
+++ b/sociofillmore/webapp/static/app.js
@@ -0,0 +1,1102 @@
+(function ($) {
+
+ // Constants
+ const datasetInfo = {
+ "femicides/rai": "This dataset consists of news reports and metadata about femicides (i.e., murders in which the perpetrator, often a partner or ex-partner, kills a woman because of her gender) perpetrated in Italy between 2015-2017. The dataset has been compiled by the research unit of national public broadcaster RAI in collaboration with a team of sociologists. The news reports come from a variety of national and regional news outlets. Because of the rich set of available metadata (only a small part of which is, as of now, available in this tool), newspaper reports that report the same femicide can be linked, allowing for investigating how one and the same event is framed in different news reports, and how framing changes over time.",
+ "femicides/olv": "This dataset consists of news reports and metadata about femicides (i.e., murders in which the perpetrator, often a partner or ex-partner, kills a woman because of her gender) perpetrated in Italy between 2012-2019. The dataset consists of a set of case summaries and metadata (see https://27esimaora.corriere.it/la-strage-delle-donne/) compiled by journalists of Corriere della Sera, a major national newspaper, and a set of newspaper articles from several national newspapers mentioning these cases (automatically matched based on victim names).",
+ "migration/pavia": "This dataset consists of immigration-related news articles in Italy published between 2013-2021, collected by the Osservatorio di Pavia media research organization (https://www.osservatorio.it/) ",
+ "crashes/thecrashes": "This dataset consists of news articles in Dutch-language news outlets (mostly regional Dutch and Belgian newspapers) describing traffic crashes, together with metadata about the events. This dataset is developed by researchers at the University of Amsterdam, in with journalists at The Correspondent, and is publically available at https://nl.roaddanger.org/."
+ };
+ const datasetFrameInfo = {
+ "femicides/rai": "For investigating the conceptualization of femicides in the media, we have created three clusters of frames. The most important one, 'murder', comprises frames that describe the act of murder itself in various degrees of agentivity (KILLING implies an active agent, DEATH and DEAD_OR_ALIVE describe the death of the victim, and CATASTROPHE and EVENT present the murders as abstract events or tragedies).",
+ "femicides/olv": "For investigating the conceptualization of femicides in the media, we have created three clusters of frames. The most important one, 'murder', comprises frames that describe the act of murder itself in various degrees of agentivity (KILLING implies an active agent, DEATH and DEAD_OR_ALIVE describe the death of the victim, and CATASTROPHE and EVENT present the murders as abstract events or tragedies).",
+ "migration/pavia": "Several clusters of frames were created, reflecting different aspects of media coverage on migration, including the depiction of migrants as quantities ('a tsunami of migrants'), political aspects of migration, and integration of migrants into society.",
+ "crashes/thecrashes": "We defined a single cluster of frames that conceptualize several salient properties of traffic crashes, both from an agent/cause-centered perspective (KILLING, CAUSE_HARM) and from a non-agentive perspective (EVENT, EXPERIENCE_BODILY_HARM)"
+ };
+
+
+ /* === GLOBAL OBJECTS === */
+ /* --- global data/parameters --- */
+ // active dataset & model
+ let activeDataset = "femicides/olv"; // should be one of: "femicides/rai", "femicides/olv", "crashes/thecrashes", "migration/pavia"
+ let activeLomeModel = "lome_0shot"; // should be one of: "lome_0shot", "lome_evalita_plus_fn", "lome_zs-tgt_ev-frm"
+
+ // list of relevant frames
+ let relevantFrameCluster = "ALL";
+ let relevantFrames = {};
+
+ // list of active filters
+ let activeEventFilters = [];
+ let activeDocumentFilters = [];
+
+ // set of filters listed in the table
+ let tableFilters = new Set();
+
+ /* --- global jQuery objects --- */
+ // div for writing the output
+ const $framesOut = $("#frames-out");
+
+ // list of relevant frames (buttons)
+ const $frameList = $("#frame-list");
+
+ // list of possible documents
+ const $documentList = $("#document-list");
+
+ // statistics table rows
+ const $tableModalRows = $("#table-modal-rows");
+
+ // Which filters apply to which datasets?
+ const filtersToDatasets = {
+ "#doc-filter-days-after": ["rai", "olv", "thecrashes"],
+ "#doc-filter-provider": ["rai", "olv", "thecrashes", "pavia"],
+ "#doc-filter-politics-unipv": ["rai"],
+ "#doc-filter-politics-tc": ["rai"],
+ "#doc-filter-politics-agg": ["rai"],
+ "#doc-filter-politics-sc": ["pavia"],
+ "#doc-filter-religion-sc": ["pavia"],
+ "#doc-filter-type": ["rai"],
+ "#doc-filter-area": ["rai", "thecrashes"],
+ "#doc-filter-country": ["thecrashes"],
+ "#doc-filter-province": ["thecrashes"],
+ "#doc-filter-content-type": ["thecrashes"],
+ "#doc-filter-medium-type": ["thecrashes"],
+ "#doc-filter-owner": ["thecrashes"],
+ "#ev-filter-event-cat": ["rai"],
+ "#ev-filter-region": ["rai", "olv"],
+ "#ev-filter-semloc": ["rai"],
+ "#ev-filter-vict-age": ["rai", "olv"],
+ "#ev-filter-vict-nat": ["rai"],
+ "#ev-filter-injured-total": ["thecrashes"],
+ "#ev-filter-injured-child": ["thecrashes"],
+ "#ev-filter-injured-pedestrian": ["thecrashes"],
+ "#ev-filter-injured-cyclist": ["thecrashes"],
+ "#ev-filter-injured-cyclistpedestrian": ["thecrashes"],
+ "#ev-filter-injured-vehicle": ["thecrashes"],
+ "#ev-filter-dead-total": ["thecrashes"],
+ "#ev-filter-dead-child": ["thecrashes"],
+ "#ev-filter-dead-pedestrian": ["thecrashes"],
+ "#ev-filter-dead-cyclist": ["thecrashes"],
+ "#ev-filter-dead-cyclistpedestrian": ["thecrashes"],
+ "#ev-filter-dead-vehicle": ["thecrashes"],
+ "#ev-filter-deadinjured-total": ["thecrashes"],
+ "#ev-filter-deadinjured-child": ["thecrashes"],
+ "#ev-filter-deadinjured-pedestrian": ["thecrashes"],
+ "#ev-filter-deadinjured-cyclist": ["thecrashes"],
+ "#ev-filter-deadinjured-cyclistpedestrian": ["thecrashes"],
+ "#ev-filter-deadinjured-vehicle": ["thecrashes"],
+ "#ev-filter-imbalanced": ["thecrashes"],
+ "#ev-filter-att-nat": ["rai"],
+ "#ev-filter-vict-occ": ["rai"],
+ "#ev-filter-att-occ": ["rai"]
+ }
+
+ /* === FUNCTIONS: GETTING DATA FROM THE FLASK API === */
+
+ /* --- static data for feeding the option menus --- */
+ function getPossibleConstructions() {
+ const $constructionSampleList = $("#construction-sample-list, #construction-freq-select");
+ $constructionSampleList.empty();
+ $constructionSampleList.append($("