changing naming convention

cleaning/cleaning.py

@@ -0,0 +1,119 @@
+import regex
+import pandas as pd
+import json
+import sentence_transformers.util
+import os
+
+def main(raw_metadata_df, path_to_embeddings):
+    clean_metadata_df = pd.DataFrame(
+        columns=['sentences','authors','msc_tags','msc_cos_sim']
+        )
+
+    clean_title = raw_metadata_df.title.apply(cleanse)
+    clean_abstract = raw_metadata_df.summary.apply(cleanse)
+    clean_metadata_df.sentences = clean_title + ' ' + clean_abstract
+    clean_metadata_df.authors = raw_metadata_df.authors
+    clean_metadata_df.msc_tags = raw_metadata_df.categories.apply(cats_to_msc)
+
+    return clean_metadata_df
+
+##
+
+
+## 1. Latin-ize latex accents enclosed in brackets
+def remove_latex_accents(string):
+    accent = r'\\[\'\"\^\`H\~ckl=bdruvtoi]\{([a-z])\}'
+    replacement = r'\1'
+
+    string = regex.sub(accent,replacement, string)
+    return string
+
+## 2. Remove latex environments
+def remove_env(string):
+    env = r'\\[a-z]{2,}{[^{}]+?}'
+
+    string = regex.sub(env,'',string)
+    return string
+
+## 3. Latin-ize non-{} enclosed latex accents:
+def remove_accents(string):
+    accent = r'\\[\'\"\^\`H\~ckl=bdruvtoi]([a-z])'
+    replacement = r'\1'
+
+    string = regex.sub(accent,replacement,string)
+    return string 
+
+## 4. ONLY remove latex'd math that is separated as a 'word' i.e. has space characters on either side of it.
+
+def remove_latex(string):
+    latex = r'\s(\$\$?)[^\$]*?\1\S*'
+    string = regex.sub(latex,' LATEX ',string)
+    return string 
+
+
+def cleanse(string):
+    string = string.replace('\n',' ')
+    string = remove_latex_accents(string)
+    string = remove_env(string)
+    string = remove_accents(string)
+    string = remove_latex(string)
+    return string
+
+## 
+
+def find_msc(cat_list):
+    pattern = r'\b\d{2}[0-9a-zA-Z]{3}\b'
+    out = []
+    for cat in cat_list:
+        tags = regex.findall(pattern,cat)
+        for tag in tags:
+            out.append(tag)
+    return out
+
+def msc_tags():
+    with open('./data/msc.json','r') as file:
+        text = file.read()
+        return json.loads(text)
+
+
+def cats_to_msc(cat_list):
+    out = []
+    for tag in find_msc(cat_list):
+        if tag in msc_tags().keys():
+            out.append(msc_tags()[tag])
+        else:
+            continue
+    if out == []:
+        return None
+    else:
+        return out
+    
+
+##
+
+def msc_encoded_dict():
+    encoded_tags = pd.read_parquet('./data/msc_mini_embeddings.parquet').to_numpy()
+    return {k : v for (k,v) in zip(msc_tags().values(), encoded_tags)}
+
+def doc_encoded_dict():
+    library_embeddings = pd.read_parquet('./data/APSP_mini_vec.parquet')
+
+    docs = library_embeddings.docs.to_list()
+    encoded_docs = library_embeddings.vecs.to_numpy()
+
+    return {k : v for (k,v) in zip(docs , encoded_docs)}
+
+def score_tags(processed_arxiv_row):
+
+    tag_list = processed_arxiv_row.msc_tags
+    title_plus_abstract = processed_arxiv_row.docs
+
+    if tag_list is None:
+        return None
+    embedded_msc_tags = [msc_encoded_dict()[tag] for tag in tag_list]
+    
+    return sentence_transformers.util.semantic_search(
+        query_embeddings=doc_encoded_dict()[title_plus_abstract],
+        corpus_embeddings=embedded_msc_tags,
+        )[0]
+    

collection.ipynb

@@ -35813,7 +35813,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from cleaning import cleaning\n",
+    "import pandas as pd\n",
+    "import importlib\n",
+    "importlib.reload(cleaning)\n",
+    "\n",
+    "data = pd.read_parquet('./data/APSP.parquet')\n",
+    "\n",
+    "clean_data = cleaning.main(\n",
+    "    raw_arxiv_results=data,path_to_embeddings='./data/APSP_mini_vec.parquet'\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -35837,14 +35855,11 @@
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
-       "      <th>title</th>\n",
-       "      <th>summary</th>\n",
+       "      <th>clean_title</th>\n",
+       "      <th>clean_abstract</th>\n",
        "      <th>authors</th>\n",
-       "      <th>primary_category</th>\n",
-       "      <th>categories</th>\n",
-       "      <th>hyph_in_summary</th>\n",
-       "      <th>hyph_in_title</th>\n",
        "      <th>msc_tags</th>\n",
+       "      <th>msc_cos_sim</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -35853,87 +35868,90 @@
        "      <td>The Laplace spectrum on conformally compact manifolds</td>\n",
        "      <td>We consider the spectrum of the Laplace operator acting on LATEX  over a conformally compact manifold for LATEX  We prove that for LATEX  this spectrum always contains an open region of the complex plane. We further show that the spectrum is contained within a certain parabolic region of the complex plane. These regions depend on the value of LATEX  the dimension of the manifold, and the values of the sectional curvatures approaching the boundary.</td>\n",
        "      <td>[Nelia Charalambous, Julie Rowlett]</td>\n",
-       "      <td>math.SP</td>\n",
-       "      <td>[math.SP, 58c40]</td>\n",
-       "      <td>None</td>\n",
-       "      <td>None</td>\n",
        "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>On the inviscid limit connecting Brinkman's and Darcy's models of tissue growth with nonlinear pressure</td>\n",
        "      <td>Several recent papers have addressed modelling of the tissue growth by the multi-phase models where the velocity is related to the pressure by one of the physical laws (Stoke's, Brinkman's or Darcy's). While each of these models has been extensively studied, not so much is known about the connection between them. In the recent paper (arXiv:2303.10620), assuming the linear form of the pressure, the Authors connected two multi-phase models by an inviscid limit: the viscoelastic one (of Brinkman's type) and the inviscid one (of Darcy's type). Here, we prove that the same is true for a nonlinear, power-law pressure. The new ingredient is that we use relation between the pressure LATEX  and the Brinkman potential LATEX  to deduce compactness in space of LATEX  from the compactness in space of LATEX</td>\n",
        "      <td>[Charles Elbar, Jakub Skrzeczkowski]</td>\n",
-       "      <td>math.AP</td>\n",
-       "      <td>[math.AP, 35K45, 35K65, 35J60, 35Q92, 92C10]</td>\n",
-       "      <td>[power-law, multi-phase]</td>\n",
-       "      <td>None</td>\n",
-       "      <td>[Initial value problems for second-order parabolic systems]</td>\n",
+       "      <td>[Initial value problems for second-order parabolic systems, Degenerate parabolic equations, Nonlinear elliptic equations, Biomechanics]</td>\n",
+       "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>A sparse approximation of the Lieb functional with moment constraints</td>\n",
        "      <td>The aim of this paper is to present new sparsity results about the so-called Lieb functional, which is a key quantity in Density Functional Theory for electronic structure calculations for molecules. The Lieb functional was actually shown by Lieb to be a convexification of the so-called Levy-Lieb functional. Given an electronic density for a system of LATEX  electrons, which may be seen as a probability density on LATEX  the value of the Lieb functional for this density is defined as the solution of a quantum multi-marginal optimal transport problem, which reads as a minimization problem defined on the set of trace-class operators acting on the space of electronic wave-functions that are anti-symmetric LATEX  functions of LATEX  with partial trace equal to the prescribed electronic density. We introduce a relaxation of this quantum optimal transport problem where the full partial trace constraint is replaced by a finite number of moment constraints on the partial trace of the set of operators. We show that, under mild assumptions on the electronic density, there exist sparse minimizers to the resulting moment constrained approximation of the Lieb (MCAL) functional that read as operators with rank at most equal to the number of moment constraints. We also prove under appropriate assumptions on the set of moment functions that the value of the MCAL functional converges to the value of the exact Lieb functional as the number of moments go to infinity. We also prove some rates of convergence on the associated approximation of the ground state energy. We finally study the mathematical properties of the associated dual problem.</td>\n",
        "      <td>[Virginie Ehrlacher, Luca Nenna]</td>\n",
-       "      <td>math-ph</td>\n",
-       "      <td>[math-ph, math.MP, math.OC, math.SP]</td>\n",
-       "      <td>[so-called, anti-symmetric, Levy-Lieb, trace-class, multi-marginal, wave-functions]</td>\n",
-       "      <td>None</td>\n",
        "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>Stationarity and Fredholm theory in subextremal Kerr-de Sitter spacetimes</td>\n",
        "      <td>In a recent paper, we proved that solutions to linear wave equations in a subextremal Kerr-de Sitter spacetime have asymptotic expansions in quasinormal modes up to a decay order given by the normally hyperbolic trapping, extending the existing results. One central ingredient in the argument was a new definition of quasinormal modes, where a non-standard choice of stationary Killing vector field had to be used in order for the Fredholm theory to be applicable. In this paper, we show that there is in fact a variety of allowed choices of stationary Killing vector fields. In particular, the horizon Killing vector fields work for the analysis, in which case one of the corresponding ergoregions is completely removed.</td>\n",
        "      <td>[Oliver Petersen, András Vasy]</td>\n",
-       "      <td>math.AP</td>\n",
-       "      <td>[math.AP, gr-qc, math.DG, 35L05, 35P25, 58J45, 83C30]</td>\n",
-       "      <td>[non-standard, Kerr-de]</td>\n",
-       "      <td>[Kerr-de]</td>\n",
-       "      <td>[Wave equation]</td>\n",
+       "      <td>[Wave equation, Scattering theory for PDEs, Hyperbolic equations on manifolds]</td>\n",
+       "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>Remarks on paper \"Two-term spectral asymptotics in linear elasticity''</td>\n",
        "      <td>In this note, by pointing out several serious mistakes in  we show that the conclusions published by Matteo Capoferri, Leonid Friedlander, Michael Levitin and Dmitri Vassiliev (J Geom Anal (2023)33:242) are completely wrong. Then, we explain the correctness of proof of Theorem 1.1 in our paper  by giving some remarks and putting the whole proof in Appendix (see also  and ).</td>\n",
        "      <td>[Genqian Liu]</td>\n",
-       "      <td>math.SP</td>\n",
-       "      <td>[math.SP, math-ph, math.AP, math.DG, math.MP]</td>\n",
-       "      <td>None</td>\n",
-       "      <td>[Two-term]</td>\n",
        "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "                                                                                                     title  ...                                                     msc_tags\n",
-       "0  The Laplace spectrum on conformally compact manifolds                                                    ...  None                                                       \n",
-       "1  On the inviscid limit connecting Brinkman's and Darcy's models of tissue growth with nonlinear pressure  ...  [Initial value problems for second-order parabolic systems]\n",
-       "2  A sparse approximation of the Lieb functional with moment constraints                                    ...  None                                                       \n",
-       "3  Stationarity and Fredholm theory in subextremal Kerr-de Sitter spacetimes                                ...  [Wave equation]                                            \n",
-       "4  Remarks on paper \"Two-term spectral asymptotics in linear elasticity''                                   ...  None                                                       \n",
+       "                                                                                               clean_title  \\\n",
+       "0  The Laplace spectrum on conformally compact manifolds                                                     \n",
+       "1  On the inviscid limit connecting Brinkman's and Darcy's models of tissue growth with nonlinear pressure   \n",
+       "2  A sparse approximation of the Lieb functional with moment constraints                                     \n",
+       "3  Stationarity and Fredholm theory in subextremal Kerr-de Sitter spacetimes                                 \n",
+       "4  Remarks on paper \"Two-term spectral asymptotics in linear elasticity''                                    \n",
        "\n",
-       "[5 rows x 8 columns]"
+       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      clean_abstract  \\\n",
+       "0  We consider the spectrum of the Laplace operator acting on LATEX  over a conformally compact manifold for LATEX  We prove that for LATEX  this spectrum always contains an open region of the complex plane. We further show that the spectrum is contained within a certain parabolic region of the complex plane. These regions depend on the value of LATEX  the dimension of the manifold, and the values of the sectional curvatures approaching the boundary.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
+       "1  Several recent papers have addressed modelling of the tissue growth by the multi-phase models where the velocity is related to the pressure by one of the physical laws (Stoke's, Brinkman's or Darcy's). While each of these models has been extensively studied, not so much is known about the connection between them. In the recent paper (arXiv:2303.10620), assuming the linear form of the pressure, the Authors connected two multi-phase models by an inviscid limit: the viscoelastic one (of Brinkman's type) and the inviscid one (of Darcy's type). Here, we prove that the same is true for a nonlinear, power-law pressure. The new ingredient is that we use relation between the pressure LATEX  and the Brinkman potential LATEX  to deduce compactness in space of LATEX  from the compactness in space of LATEX                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
+       "2  The aim of this paper is to present new sparsity results about the so-called Lieb functional, which is a key quantity in Density Functional Theory for electronic structure calculations for molecules. The Lieb functional was actually shown by Lieb to be a convexification of the so-called Levy-Lieb functional. Given an electronic density for a system of LATEX  electrons, which may be seen as a probability density on LATEX  the value of the Lieb functional for this density is defined as the solution of a quantum multi-marginal optimal transport problem, which reads as a minimization problem defined on the set of trace-class operators acting on the space of electronic wave-functions that are anti-symmetric LATEX  functions of LATEX  with partial trace equal to the prescribed electronic density. We introduce a relaxation of this quantum optimal transport problem where the full partial trace constraint is replaced by a finite number of moment constraints on the partial trace of the set of operators. We show that, under mild assumptions on the electronic density, there exist sparse minimizers to the resulting moment constrained approximation of the Lieb (MCAL) functional that read as operators with rank at most equal to the number of moment constraints. We also prove under appropriate assumptions on the set of moment functions that the value of the MCAL functional converges to the value of the exact Lieb functional as the number of moments go to infinity. We also prove some rates of convergence on the associated approximation of the ground state energy. We finally study the mathematical properties of the associated dual problem.   \n",
+       "3  In a recent paper, we proved that solutions to linear wave equations in a subextremal Kerr-de Sitter spacetime have asymptotic expansions in quasinormal modes up to a decay order given by the normally hyperbolic trapping, extending the existing results. One central ingredient in the argument was a new definition of quasinormal modes, where a non-standard choice of stationary Killing vector field had to be used in order for the Fredholm theory to be applicable. In this paper, we show that there is in fact a variety of allowed choices of stationary Killing vector fields. In particular, the horizon Killing vector fields work for the analysis, in which case one of the corresponding ergoregions is completely removed.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
+       "4  In this note, by pointing out several serious mistakes in  we show that the conclusions published by Matteo Capoferri, Leonid Friedlander, Michael Levitin and Dmitri Vassiliev (J Geom Anal (2023)33:242) are completely wrong. Then, we explain the correctness of proof of Theorem 1.1 in our paper  by giving some remarks and putting the whole proof in Appendix (see also  and ).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
+       "\n",
+       "                                authors  \\\n",
+       "0  [Nelia Charalambous, Julie Rowlett]    \n",
+       "1  [Charles Elbar, Jakub Skrzeczkowski]   \n",
+       "2  [Virginie Ehrlacher, Luca Nenna]       \n",
+       "3  [Oliver Petersen, András Vasy]         \n",
+       "4  [Genqian Liu]                          \n",
+       "\n",
+       "                                                                                                                                  msc_tags  \\\n",
+       "0  None                                                                                                                                      \n",
+       "1  [Initial value problems for second-order parabolic systems, Degenerate parabolic equations, Nonlinear elliptic equations, Biomechanics]   \n",
+       "2  None                                                                                                                                      \n",
+       "3  [Wave equation, Scattering theory for PDEs, Hyperbolic equations on manifolds]                                                            \n",
+       "4  None                                                                                                                                      \n",
+       "\n",
+       "  msc_cos_sim  \n",
+       "0  NaN         \n",
+       "1  NaN         \n",
+       "2  NaN         \n",
+       "3  NaN         \n",
+       "4  NaN         "
       ]
      },
-     "execution_count": 22,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "lib.clean_lib.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "lib.clean_lib.to_parquet('./data/clean_APSP.parquet')"
+    "pd.set_option('display.max_colwidth', 0)\n",
+    "clean_data.head()"
    ]
   }
  ],

util.py

@@ -253,10 +253,7 @@ def find_msc(cat_list):
         tags = regex.findall(pattern,cat)
         for tag in tags:
             out.append(tag)
-    if out == []:
-        return None
-    else:
-        return out
+    return out
 
 
 def format_query(author='',title='',cat='',abstract=''):