Upload 13 files

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 local_files/galaxy_worldmap_kiyer-min.png filter=lfs diff=lfs merge=lfs -text
+data/astrophindex.faiss filter=lfs diff=lfs merge=lfs -text

data/astrophindex.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10089ae87900eb8c9f1698c36bb5d1128d712b7b90f3f3d07f074ab130027440
+size 2163879981

data/data-00000-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fd656a426f8ff3d7c2bb7164154b042d50f5e5deeeb7ead12b9baee7b9d5f8d
+size 509410376

data/data-00001-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:952f8125fc109ff15b7abbfb10ac928b6070087899e79ebbf62500b018a2bac7
+size 503809992

data/data-00002-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1069ba9c791822f5761b7311f69bc0323294751499988bcb3ae904b339a89dd
+size 504473320

data/data-00003-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f91fb8b3aed602e091b89e35fff7432d4573f5ccbdc2d5d9d19daabbacc3a5b
+size 508874456

data/data-00004-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb7f52083e24ef80fc12bca90c88aac3159d5479af9f29de4a7740dacb85d9d6
+size 501644912

data/data-00005-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77f69ede7a456f8f59873e482bf381626532b945e50b497b91834d814ba8ce2
+size 501508320

data/data-00006-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:762b9acd3aa46c27b40af2a681b638e6ac345fbb508360c68f815e284d61f584
+size 470994224

data/data-00007-of-00008.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58b898027e4506c6a7a0afb9bdc5e61c8df13a8d5089a68383a457dc6dea1b72
+size 497639288

data/dataset_info.json

@@ -3,59 +3,43 @@
   "citation": "",
   "config_name": "default",
   "dataset_name": "pathfinder_arxiv_data",
-  "dataset_size": 5770056875,
+  "dataset_size": 4065510154,
   "description": "",
   "download_checksums": {
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00000-of-00012.parquet": {
-      "num_bytes": 384481705,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00000-of-00008.parquet": {
+      "num_bytes": 406754152,
       "checksum": null
     },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00001-of-00012.parquet": {
-      "num_bytes": 383347319,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00001-of-00008.parquet": {
+      "num_bytes": 405109745,
       "checksum": null
     },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00002-of-00012.parquet": {
-      "num_bytes": 383133689,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00002-of-00008.parquet": {
+      "num_bytes": 405466052,
       "checksum": null
     },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00003-of-00012.parquet": {
-      "num_bytes": 384399351,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00003-of-00008.parquet": {
+      "num_bytes": 406784839,
       "checksum": null
     },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00004-of-00012.parquet": {
-      "num_bytes": 382810245,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00004-of-00008.parquet": {
+      "num_bytes": 404752067,
       "checksum": null
     },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00005-of-00012.parquet": {
-      "num_bytes": 382870394,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00005-of-00008.parquet": {
+      "num_bytes": 404624503,
       "checksum": null
     },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00006-of-00012.parquet": {
-      "num_bytes": 364849142,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00006-of-00008.parquet": {
+      "num_bytes": 392634525,
       "checksum": null
     },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00007-of-00012.parquet": {
-      "num_bytes": 363965178,
-      "checksum": null
-    },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00008-of-00012.parquet": {
-      "num_bytes": 376639054,
-      "checksum": null
-    },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00009-of-00012.parquet": {
-      "num_bytes": 384035100,
-      "checksum": null
-    },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00010-of-00012.parquet": {
-      "num_bytes": 355126903,
-      "checksum": null
-    },
-    "hf://datasets/kiyer/pathfinder_arxiv_data@66fc52fb3d7d82779c3d73b0cb0c14218cb02e63/data/train-00011-of-00012.parquet": {
-      "num_bytes": 359912183,
+    "hf://datasets/kiyer/pathfinder_arxiv_data@1a8eaa7eef5a503386a1487e20f13bedba605245/data/train-00007-of-00008.parquet": {
+      "num_bytes": 397101125,
       "checksum": null
     }
   },
-  "download_size": 4505570263,
+  "download_size": 3223227008,
   "features": {
     "ads_id": {
       "dtype": "string",
@@ -156,25 +140,22 @@
   },
   "homepage": "",
   "license": "",
-  "size_in_bytes": 10275627138,
+  "size_in_bytes": 7288737162,
   "splits": {
     "train": {
       "name": "train",
-      "num_bytes": 5770056875,
-      "num_examples": 499142,
+      "num_bytes": 4065510154,
+      "num_examples": 352194,
       "shard_lengths": [
-        42596,
-        43596,
-        43595,
-        42595,
-        43595,
-        43595,
-        46595,
-        44595,
-        43595,
-        43595,
-        43595,
-        17595
+        43000,
+        43025,
+        43025,
+        43024,
+        44024,
+        44024,
+        46024,
+        44024,
+        2024
       ],
       "dataset_name": "pathfinder_arxiv_data"
     }

data/state.json

@@ -1,43 +1,31 @@
 {
   "_data_files": [
     {
-      "filename": "data-00000-of-00012.arrow"
+      "filename": "data-00000-of-00008.arrow"
     },
     {
-      "filename": "data-00001-of-00012.arrow"
+      "filename": "data-00001-of-00008.arrow"
     },
     {
-      "filename": "data-00002-of-00012.arrow"
+      "filename": "data-00002-of-00008.arrow"
     },
     {
-      "filename": "data-00003-of-00012.arrow"
+      "filename": "data-00003-of-00008.arrow"
     },
     {
-      "filename": "data-00004-of-00012.arrow"
+      "filename": "data-00004-of-00008.arrow"
     },
     {
-      "filename": "data-00005-of-00012.arrow"
+      "filename": "data-00005-of-00008.arrow"
     },
     {
-      "filename": "data-00006-of-00012.arrow"
+      "filename": "data-00006-of-00008.arrow"
     },
     {
-      "filename": "data-00007-of-00012.arrow"
-    },
-    {
-      "filename": "data-00008-of-00012.arrow"
-    },
-    {
-      "filename": "data-00009-of-00012.arrow"
-    },
-    {
-      "filename": "data-00010-of-00012.arrow"
-    },
-    {
-      "filename": "data-00011-of-00012.arrow"
+      "filename": "data-00007-of-00008.arrow"
     }
   ],
-  "_fingerprint": "10a80a75c30e04f8",
+  "_fingerprint": "216019f3026e4d55",
   "_format_columns": null,
   "_format_kwargs": {},
   "_format_type": null,

prompts.py

@@ -0,0 +1,144 @@
+react_prompt = """You are an expert astronomer and cosmologist.
+    Answer the following question as best you can using information from the library, but speaking in a concise and factual manner.
+    If you can not come up with an answer, say you do not know.
+    Try to break the question down into smaller steps and solve it in a logical manner.
+
+    You have access to the following tools:
+
+    {tools}
+
+    Use the following format:
+
+    Question: the input question you must answer
+    Thought: you should always think about what to do
+    Action: the action to take, should be one of [{tool_names}]
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Thought/Action/Action Input/Observation can repeat N times)
+    Thought: I now know the final answer
+    Final Answer: the final answer to the original input question. provide information about how you arrived at the answer, and any nuances or uncertainties the reader should be aware of
+
+    Begin! Remember to speak in a pedagogical and factual manner."
+
+    Question: {input}
+    Thought:{agent_scratchpad}"""
+
+regular_prompt = """You are an expert astronomer and cosmologist.
+Answer the following question as best you can using information from the library, but speaking in a concise and factual manner.
+If you can not come up with an answer, say you do not know.
+Try to break the question down into smaller steps and solve it in a logical manner.
+
+Provide information about how you arrived at the answer, and any nuances or uncertainties the reader should be aware of.
+
+Begin! Remember to speak in a pedagogical and factual manner."
+
+Relevant documents:{context}
+
+Question: {question}
+Answer:"""
+
+bibliometric_prompt = """You are an AI assistant with expertise in astronomy and astrophysics literature. Your task is to assist with relevant bibliometric information in response to a user question. The user question may consist of identifying key papers, authors, or trends in a specific area of astronomical research.
+
+Depending on what the user wants, direct them to consult the NASA Astrophysics Data System (ADS) at https://ui.adsabs.harvard.edu/. Provide them with the recommended ADS query depending on their question.
+
+Here's a more detailed guide on how to use NASA ADS for various types of queries:
+
+Basic topic search: Enter keywords in the search bar, e.g., "exoplanets". Use quotation marks for exact phrases, e.g., "dark energy”
+Author search: Use the syntax author:"Last Name, First Name", e.g., author:"Hawking, S". For papers by multiple authors, use AND, e.g., author:"Hawking, S" AND author:"Ellis, G"
+Date range: Use year:YYYY-YYYY, e.g., year:2010-2020. For papers since a certain year, use year:YYYY-, e.g., year:2015-
+4.Combining search terms: Use AND, OR, NOT operators, e.g., "black holes" AND (author:"Hawking, S" OR author:"Penrose, R")
+Filtering results: Use the left sidebar to filter by publication year, article type, or astronomy database
+Sorting results: Use the "Sort" dropdown menu to order by options like citation count, publication date, or relevance
+Advanced searches: Click on the "Search" dropdown menu and select "Classic Form" for field-specific searchesUse bibcode:YYYY for a specific journal/year, e.g., bibcode:2020ApJ to find all Astrophysical Journal papers from 2020
+Finding review articles: Wrap the query in the reviews() operator (e.g. reviews(“dark energy”))
+Excluding preprints: Add NOT doctype:"eprint" to your search
+Citation metrics: Click on the citation count of a paper to see its citation history and who has cited it
+
+Some examples:
+
+Example 1:
+“How many papers published in 2022 used data from MAST missions?”
+Your response should be: year:2022  data:"MAST"
+
+Example 2:
+“What are the most cited papers on spiral galaxy halos measured in X-rays, with publication date from 2010 to 2023?
+Your response should be: "spiral galaxy halos" AND "x-ray" year:2010-2024
+
+Example 3:
+“Can you list 3 papers published by “< name>” as first author?”
+Your response should be: author: “^X”
+
+Example 4:
+“Based on papers with “<name>” as an author or co-author, can you suggest the five most recent astro-ph papers that would be relevant?”
+Your response should be:
+
+Remember to advise users that while these examples cover many common scenarios, NASA ADS has many more advanced features that can be explored through its documentation.
+
+Relevant documents:{context}
+Question: {question}
+
+Response:"""
+
+single_paper_prompt = """You are an astronomer with access to a vast database of astronomical facts and figures. Your task is to provide a concise, accurate answer to the following specific factual question about astronomy or astrophysics.
+Provide the requested information clearly and directly. If relevant, include the source of your information or any recent updates to this fact. If there's any uncertainty or variation in the accepted value, briefly explain why.
+If the question can't be answered with a single fact, provide a short, focused explanation. Always prioritize accuracy over speculation.
+Relevant documents:{context}
+Question: {question}
+Response:"""
+
+deep_knowledge_prompt = """You are an expert astronomer with deep knowledge across various subfields of astronomy and astrophysics. Your task is to provide a comprehensive and nuanced answer to the following question, which involves an unresolved topic or requires broad, common-sense understanding.
+Consider multiple perspectives and current debates in the field. Explain any uncertainties or ongoing research. If relevant, mention how this topic connects to other areas of astronomy.
+Provide your response in a clear, pedagogical manner, breaking down complex concepts for easier understanding. If appropriate, suggest areas where further research might be needed.
+After formulating your initial response, take a moment to reflect on your answer. Consider:
+1. Have you addressed all aspects of the question?
+2. Are there any potential biases or assumptions in your explanation?
+3. Is your explanation clear and accessible to someone with a general science background?
+4. Have you adequately conveyed the uncertainties or debates surrounding this topic?
+Based on this reflection, refine your answer as needed.
+Remember, while you have extensive knowledge, it's okay to acknowledge the limits of current scientific understanding. If parts of the question cannot be answered definitively, explain why.
+Relevant documents:{context}
+
+Question: {question}
+
+Initial Response:
+[Your initial response here]
+
+Reflection and Refinement:
+[Your reflections and any refinements to your answer here]
+
+Final Response:
+[Your final, refined answer here]"""
+
+question_categorization_prompt = """You are an expert astrophysicist and computer scientist specializing in linguistics and semantics. Your task is to categorize a given query into one of the following categories:
+
+1. Summarization
+2. Single-paper factual
+3. Multi-paper factual
+4. Named entity recognition
+5. Jargon-specific questions / overloaded words
+6. Time-sensitive
+7. Consensus evaluation
+8. What-ifs and counterfactuals
+9. Compositional
+
+Analyze the query carefully, considering its content, structure, and implications. Then, determine which of the above categories best fits the query.
+
+In your analysis, consider the following:
+- Does the query ask for a well-known datapoint or mechanism?
+- Can it be answered by a single paper or does it require multiple sources?
+- Does it involve proper nouns or specific scientific terms?
+- Is it time-dependent or likely to change in the near future?
+- Does it require evaluating consensus across multiple sources?
+- Is it a hypothetical or counterfactual question?
+- Does it need to be broken down into sub-queries (i.e. compositional)?
+
+After your analysis, categorize the query into one of the nine categories listed above.
+
+Provide a brief explanation for your categorization, highlighting the key aspects of the query that led to your decision.
+
+Present your final answer in the following format:
+
+<categorization>
+Category: [Selected category]
+Explanation: [Your explanation for the categorization]
+</categorization>"""