initial commit

app.py

@@ -0,0 +1,83 @@
+import gradio as gr
+from clean_bibtex.clean_bibtex import get_url, get_dblp_bibtext, parse_bibtext_file_titles
+
+DEFAULT_TEXT = """@inproceedings{DBLP:conf/naacl/DevlinCLT19,
+  author    = {Jacob Devlin and
+               Ming{-}Wei Chang and
+               Kenton Lee and
+               Kristina Toutanova},
+  editor    = {Jill Burstein and
+               Christy Doran and
+               Thamar Solorio},
+  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
+               Understanding},
+  booktitle = {Proceedings of the 2019 Conference of the North American Chapter of
+               the Association for Computational Linguistics: Human Language Technologies,
+               {NAACL-HLT} 2019, Minneapolis, MN, USA, June 2-7, 2019, Volume 1 (Long
+               and Short Papers)},
+  pages     = {4171--4186},
+  publisher = {Association for Computational Linguistics},
+  year      = {2019},
+  url       = {https://doi.org/10.18653/v1/n19-1423},
+  doi       = {10.18653/v1/n19-1423},
+  timestamp = {Fri, 06 Aug 2021 00:41:31 +0200},
+  biburl    = {https://dblp.org/rec/conf/naacl/DevlinCLT19.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+"""
+
+
+def parse_titles(bibtex):
+    titles = []
+    lines = bibtex.split("\n")
+    print(lines)
+    for line in lines:
+        if line.strip().startswith("title"):
+            title = "".join(line.split("=")[1:])
+            title_clean = title.replace("{", "").replace("}", "").replace(",\n", "").strip()
+            titles.append(title_clean)
+    return titles
+
+
+def cleaner(bibtex, keep_keys, file_obj):
+    dblp_citations = []
+    errors = []
+
+    if file_obj:
+        titles = parse_bibtext_file_titles(file_obj.name)
+
+    elif bibtex:
+        titles = parse_titles(bibtex)
+
+    for publication in titles:
+        if site_url := get_url(publication):
+            if dblp_citation := get_dblp_bibtext(site_url):
+                dblp_citations.append(dblp_citation)
+            else:
+                errors.append(" - " + publication)
+        else:
+            errors.append(" - " + publication)
+
+    if dblp_citations:
+        filenaem = "cleaned.bib"
+        with open(filenaem, "w") as outFile:
+            outFile.write("\n".join(dblp_citations))
+
+    return filenaem, "\n".join(errors)
+
+
+iface = gr.Interface(
+    fn=cleaner,
+    title="Bibtex cleaner",
+    description="Clean a bibtex file or string simply by dragging the incomplete or broken bibtex file into the file box or pasting a bibtex string into the string field. The titles are extrected, searched at the DBLP and a clean bibtexfile is created.",
+    inputs=[
+        gr.inputs.Textbox(label="Paste a string here:", lines=1),
+        gr.inputs.Checkbox(label="Keep original keys:"),
+        gr.inputs.File(label="Drag a Bibtex file here:", file_count="single", type="file", optional=True),
+    ],
+    outputs=[
+        gr.outputs.File(label="Cleaned bibtext file:"),
+        gr.outputs.Textbox(type="auto", label="Errors:"),
+    ],
+)
+iface.launch()

clean_bibtex/clean_bibtex.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+"""This python script parses an incomplete BibTeX file to a BibTeX file with dblp references and styling.
+
+Example:
+    python bibtext_to_dblp <bibtext input file> <output file>
+"""
+
+import requests
+import click
+from typing import Optional
+import time
+
+
+def parse_bibtext_file_titles(file_path: str) -> list[str]:
+    """Function to parse the titles of the publications from a BibTeX file.
+
+    Args:
+        file_path (str): File path of the BibTeX file to parse.
+
+    Returns:
+        list[str]: List with the parsed titles.
+    """
+    try:
+        titles = []
+        with open(file_path, "r") as inFile:
+            for line in inFile.readlines():
+                if line.strip().startswith("title"):
+                    title = "".join(line.split("=")[1:])
+                    title_clean = title.replace("{", "").replace("}", "").replace(",\n", "").strip()
+                    titles.append(title_clean)
+        return titles
+    except OSError as err:
+        print("OS error: {0}".format(err))
+        raise
+    except ValueError:
+        print("Could not parse, bibtext file is malformed.")
+        raise
+    except BaseException as err:
+        print(f"Unexpected {err}, {type(err)}")
+        raise
+
+
+def get_url(title: str) -> Optional[str]:
+    """Search DBLP with a publication title and parse the pdf from the best result.json.
+
+    Args:
+        title (str): Title of the publication to search for.
+
+    Returns:
+        Optional[str]: URL of the DBLP page of the publication or None.
+    """
+    url = f"https://dblp.org/search/publ/api?q={title}&format=json"
+    result = requests.get(url)
+
+    try:
+        url = result.json()["result"]["hits"]["hit"][0]["info"]["url"]
+        return url
+    except:
+        return None
+
+
+def get_dblp_bibtext(url: str) -> Optional[str]:
+    """Get the bibtext reference from a dblp publikation site url.
+
+    Args:
+        url (str): Url to the publication site.
+
+    Returns:
+        Optional[str]: Bibtex reference for the publication or None if an error occurred.
+    """
+    r = requests.get(url + ".bib")
+    if r.status_code == 200:
+        return r.text
+    else:
+        return None
+
+
+@click.command()
+@click.argument("input_file")
+@click.argument("outpu_file")
+def clean_bibtex(outpu_file: str, input_file: str):
+    """Convert an incomplete BibTeX file into a complete BibTeX file with dblp styling.
+
+    Args:
+        outpu_file (str): Destination for the new file.
+        input_file (str): Input file to parse bibtext citations from.
+    """
+    titles = parse_bibtext_file_titles(input_file)
+    errors = []
+    num_publications = str(len(titles))
+
+    click.echo("Requesting citation metadata for {num_publications} publications, this may take a while...")
+    with click.progressbar(length=len(titles)) as bar:
+        dblp_citations = []
+        for publication in titles:
+            if site_url := get_url(publication):
+                if dblp_citation := get_dblp_bibtext(site_url):
+                    dblp_citations.append(dblp_citation)
+                else:
+                    errors.append(" - " + publication)
+            else:
+                errors.append(" - " + publication)
+            time.sleep(1)  # abide dblp crawl-delay
+            bar.update(1)
+
+    if dblp_citations:
+        with open(outpu_file, "w") as outFile:
+            outFile.write("\n".join(dblp_citations))
+        click.echo(f"\nNew BibTeX file written to: {outpu_file}")
+    else:
+        click.echo("No citations to write.")
+    if errors:
+        click.echo("\nCould not create citations for:")
+        click.echo("\n".join(errors))
+
+
+if __name__ == "__main__":
+    clean_bibtex()

requirements.txt

@@ -0,0 +1,2 @@
+requests
+gradio