Spaces:
Sleeping
Sleeping
File size: 26,036 Bytes
5e5d326 2099574 ef26f24 5e5d326 5db794d 5e5d326 ef26f24 5e5d326 aa7ef71 5e5d326 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 |
import os
TOGETHER_API_KEY = os.environ.get("TOGETHER_API_KEY")
SEMANTIC_SCHOLAR_API_KEY = os.environ.get("SEMANTIC_SCHOLAR_API_KEY")
import re
import time
import json
import shutil
import requests
import spacy
#!python -m spacy download en_core_web_lg
from openai import OpenAI, APIError
from llama_index import (
VectorStoreIndex,
SimpleDirectoryReader,
ServiceContext,
load_index_from_storage
)
from llama_index.embeddings import HuggingFaceEmbedding, TogetherEmbedding
from llama_index.storage.storage_context import StorageContext
# 处理原始.text数据抹去citation,或者直接从用户出获得没有citation的introduct
def remove_citation(text):
# Regular expression to match \cite{...}
pattern = r'\\cite\{[^}]*\}'
# Replace \cite{...} with an empty string
text = re.sub(pattern, '', text)
# Replace multiple spaces with a single space
text = re.sub(r' +', ' ', text)
# Replace spaces before punctuation marks with just the punctuation marks
text = re.sub(r"\s+([,.!?;:()\[\]{}])", r"\1", text)
return text
def get_chat_completion(client, prompt, llm_model, max_tokens):
messages = [
{
"role": "system",
"content": "You are an AI assistant",
},
{
"role": "user",
"content": prompt,
}
]
try:
chat_completion = client.chat.completions.create(
messages=messages,
model=llm_model,
max_tokens=max_tokens
)
return chat_completion.choices[0].message.content
except APIError as e:
# Handle specific API errors
print(f"API Error: {e}")
except Exception as e:
# Handle other exceptions
print(f"Error: {e}")
def get_relevant_papers(search_query, sort=True, count=10):
"""
search_query (str): the required query parameter and its value (in this case, the keyword we want to search for)
count (int): the number of relevant papers to return for each query
Semantic Scholar Rate limit:
1 request per second for the following endpoints:
/paper/batch
/paper/search
/recommendations
10 requests / second for all other calls
"""
# Define the paper search endpoint URL; All keywords in the search query are matched against the paper’s title and abstract.
url = 'https://api.semanticscholar.org/graph/v1/paper/search'
# Define headers with API key
headers = {'x-api-key': SEMANTIC_SCHOLAR_API_KEY}
query_params = {
'query': search_query,
'fields': 'url,title,year,abstract,authors.name,journal,citationStyles,tldr,referenceCount,citationCount',
'limit': 20,
}
# Send the API request
response = requests.get(url, params=query_params, headers=headers)
# Check response status
if response.status_code == 200:
json_response = response.json()
if json_response['total'] != 0:
papers = json_response['data']
else:
papers = []
# Sort the papers based on citationCount in descending order
if sort:
papers = sorted(papers, key=lambda x: x['citationCount'], reverse=True)
return papers[:count]
else:
print(f"Request failed with status code {response.status_code}: {response.text}")
def save_papers(unique_dir, papers):
os.makedirs(unique_dir, exist_ok=True)
# Save each dictionary to a separate JSON file
for i, dictionary in enumerate(papers):
filename = os.path.join(unique_dir, f"{dictionary['paperId']}.json")
with open(filename, 'w') as json_file:
json.dump(dictionary, json_file, indent=4)
print(f"{len(papers)} papers saved as JSON files successfully at {unique_dir}.")
def get_index(service_context, docs_dir, persist_dir):
documents = SimpleDirectoryReader(docs_dir, filename_as_id=True).load_data()
# check if storage already exists
PERSIST_DIR = persist_dir
if not os.path.exists(PERSIST_DIR):
print('create new index')
index = VectorStoreIndex.from_documents(
documents, service_context=service_context, show_progress=False
)
# store it for later
index.storage_context.persist(persist_dir=PERSIST_DIR)
else:
print('load the existing index')
# load the existing index
storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
index = load_index_from_storage(storage_context, service_context=service_context)
# refresh the index
refreshed_docs = index.refresh_ref_docs(documents, update_kwargs={"delete_kwargs": {"delete_from_docstore": True}})
print(f'refreshed_docs:\n{refreshed_docs}')
return index
def get_paper_data(text):
"""text = node.text """
dictionary_from_json = json.loads(text)
bibtex = dictionary_from_json['citationStyles']['bibtex']
bibtex = bibtex.replace('&', 'and')
citation_label = re.findall(r'@(\w+){([\w-]+)', bibtex)[0][1]
citationCount = dictionary_from_json['citationCount']
if dictionary_from_json['tldr'] is not None:
tldr = dictionary_from_json['tldr']['text']
else:
tldr = 'No tldr available'
url = dictionary_from_json['url']
return citation_label, (bibtex, citationCount, tldr, url)
def move_cite_inside_sentence(sent, ez_citation):
if sent[-1]!='\n':
character = sent[-1]
sent_new = sent[:-1] + ' <ez_citation>' + character
else:
count = sent.count('\n')
character = sent[-(count+1)]
sent_new = sent[:-(count+1)] + ' <ez_citation>' + character + '\n'*count
return sent_new.replace('<ez_citation>', ez_citation)
def write_bib_file(bib_file_content, data):
bibtex, citationCount, tldr, url = data
bib_file_content = bib_file_content + f'\n%citationCount: {citationCount}\n%tldr: {tldr}\n%url: {url}\n' + bibtex
return bib_file_content
def write_citation(sent, bib_file_content, retrieved_nodes, sim_threshold=0.75):
labels = []
for node in retrieved_nodes:
citation_label, data = get_paper_data(node.text)
print('relevant paper id (node.id_):', node.id_, 'match score (node.score):', node.score)
print('relevant paper data:', *data)
print('-'*30)
if node.score > sim_threshold and citation_label != "None":
labels.append(citation_label)
if not (citation_label in bib_file_content):
bib_file_content = write_bib_file(bib_file_content, data)
else:
continue
labels = ', '.join(labels)
if labels:
ez_citation = f'\cite{{{labels}}}'
sent_new = move_cite_inside_sentence(sent, ez_citation)
else:
sent_new = sent
return sent_new, bib_file_content
get_prompt = lambda sentence: f"""
I want to use semantic scholar paper search api to find the relevant papers, can you read the following text then suggest me an suitable search query for this task?
Here is an example for using the api:
<example>
```python
import requests
# Define the paper search endpoint URL
url = 'https://api.semanticscholar.org/graph/v1/paper/search'
# Define the required query parameter and its value (in this case, the keyword we want to search for)
query_params = {{
'query': 'semantic scholar platform',
'limit': 3
}}
# Make the GET request with the URL and query parameters
searchResponse = requests.get(url, params=query_params)
```
</example>
Here is the text:
<text>
{sentence}
</text>
"""
# main block
def main(sentences, count, client, llm_model, max_tokens, service_context):
"""count (int): the number of relevant papers to return for each query"""
sentences_new = []
bib_file_content = ''
for sentence in sentences:
prompt = get_prompt(sentence)
response = get_chat_completion(client, prompt, llm_model, max_tokens)
# Define a regular expression pattern to find the value of 'query'
pattern = r"'query': '(.*?)'"
matches = re.findall(pattern, response)
if matches:
search_query = matches[0]
else:
search_query = sentence[:2] # use the first two words as the search query
relevant_papers = get_relevant_papers(search_query, sort=True, count=count)
if relevant_papers:
# save papers to json files and build index
unique_dir = os.path.join("papers", f"{int(time.time())}")
persist_dir = os.path.join("index", f"{int(time.time())}")
save_papers(unique_dir, relevant_papers)
index = get_index(service_context, unique_dir, persist_dir)
# get sentence's most similar papers
retriever = index.as_retriever(service_context=service_context, similarity_top_k=5)
retrieved_nodes = retriever.retrieve(sentence)
sent_new, bib_file_content = write_citation(sentence, bib_file_content, retrieved_nodes, sim_threshold=0.7)
sentences_new.append(sent_new)
else:
sentences_new.append(sentence)
print('sentence:', sentence.strip())
print('search_query:', search_query)
print('='*30)
return sentences_new, bib_file_content
def ez_cite(introduction, debug=False):
nlp = spacy.load("en_core_web_lg")
doc = nlp(introduction)
sentences = [sentence.text for sentence in doc.sents]
sentences = [ remove_citation(sentence) for sentence in sentences]
client = OpenAI(api_key=TOGETHER_API_KEY,
base_url='https://api.together.xyz',
)
llm_model = "Qwen/Qwen1.5-72B-Chat"
max_tokens = 1000
embed_model = TogetherEmbedding(model_name="togethercomputer/m2-bert-80M-8k-retrieval", api_key=TOGETHER_API_KEY)
service_context = ServiceContext.from_defaults(
llm=None, embed_model=embed_model, chunk_size=8192, # chunk_size must be bigger than the whole .json so that all info is preserved, in this case, one doc is one node
)
if debug:
sentences = sentences[:2]
sentences_new, bib_file_content = main(sentences, count=10,
client=client,
llm_model=llm_model,
max_tokens=max_tokens,
service_context=service_context)
with open('intro.bib', 'w') as bib_file:
bib_file.write(bib_file_content)
final_intro = ' '.join(sentences_new)
print(final_intro)
print('='*30)
dir_path = "index"
try:
# Delete the directory and its contents
shutil.rmtree(dir_path)
print(f"Directory '{dir_path}' deleted successfully.")
except Exception as e:
print(f"Error deleting directory '{dir_path}': {e}")
dir_path = "papers"
try:
# Delete the directory and its contents
shutil.rmtree(dir_path)
print(f"Directory '{dir_path}' deleted successfully.")
except Exception as e:
print(f"Error deleting directory '{dir_path}': {e}")
return final_intro, bib_file_content
# arXiv:2209.05523v2
introduction = r"""A long-standing paradigm in machine learning is the trade-off between the complexity of a model family and the model's ability to generalize: more expressive model classes contain better candidates to fit complex trends in data, but are also prone to overfitting noise \cite{nielsen2015neural, geman1992neural}. \textit{Interpolation}, defined for our purposes as choosing a model with zero training error, was hence long considered bad practice \cite{hastie2009elements}. The success of deep learning - machine learning in a specific regime of extremely complex model families with vast amounts of tunable parameters - seems to contradict this notion; here, consistent evidence shows that among some interpolating models, more complexity tends \textit{not to harm} the generalisation performance, a phenomenon described as "benign overfitting" \cite{bartlett2021deep}.
In recent years, a surge of theoretical studies have reproduced benign overfitting in simplified settings with the hope of isolating the essential ingredients of the phenomenon \cite{bartlett2021deep, belkin2021fit}. For example, Ref. \cite{bartlett2020benign} showed how interpolating linear models in a high complexity regime (more dimensions than datapoints) could generalize just as well as their lower-complexity counterparts on new data, and analyzed the properties of the data that lead to the "absorption" of noise by the interpolating model without harming the model's predictions. Ref. \cite{belkin2019reconciling} showed that there are model classes of simple functions that change quickly in the vicinity of the noisy training data, but recover a smooth trend elsewhere in data space (see Figure 1). Such functions have also been used to train nearest neighbor models that perfectly overfit training data while generalizing well, thereby directly linking "spiking models" to benign overfitting \cite{belkin2019does}. Recent works try to recover the basic mechanism of such spiking models using the language of Fourier analysis \cite{muthukumar2020harmless, muthukumar2021classification, dar2021farewell}.
In parallel to these exciting developments in the theory of deep learning, quantum computing researchers have proposed families of parametrised quantum algorithms as model classes for machine learning (e.g. Ref. \cite{benedetti2019parameterized}). These quantum models can be optimised similarly to neural networks \cite{mitarai2018quantum, schuld2019evaluating} and have interesting parallels to kernel methods \cite{schuld2019quantum, havlivcek2019supervised} and generative models \cite{lloyd2018quantum, dallaire2018quantum}. Although researchers have taken some first steps to study the expressivity \cite{abbas2021power, wright2020capacity, sim2019expressibility, hubregtsen2021evaluation}, trainability \cite{mcclean2018barren, cerezo2021cost} and generalisation \cite{caro2021encoding, huang_power_2021, caro2022generalization, banchi2021generalization} of quantum models, we still know relatively little about their behaviour. In particular, the interplay of overparametrisation, interpolation, and generalisation that seems so important for deep learning is yet largely unexplored.
In this paper we develop a simplified framework in which questions of overfitting in quantum machine learning can be investigated. Essentially, we exploit the observation that quantum models can often be described in terms of Fourier series where well-defined components of the quantum circuit influence the selection of Fourier modes and their respective Fourier coefficients \cite{gil2020input, schuld2021effect, wierichs2022general}. We link this description to the analysis of spiking models and benign overfitting by building on prior works analyzing these phenomena using Fourier methods. In this approach, the complexity of a model is related to the number of Fourier modes that its Fourier series representation consists of, and overparametrised model classes have more modes than needed to interpolate the training data (i.e., to have zero training error). After deriving the generalization error for such model classes these "superfluous" modes lead to spiking models, which have large oscillations around the training data while keeping a smooth trend everywhere else. However, large numbers of modes can also harm the recovery of an underlying signal, and we therefore balance this trade-off to produce an explicit example of benign overfitting in a quantum machine learning model.
The mathematical link described above allows us to probe the impact of important design choices for a simplified class of quantum models on this trade-off. For example, we find why a measure of redundancy in the spectrum of the Hamiltonian that defines standard data encoding strategies strongly influences this balance; in fact to an extent that is difficult to counterbalance by other design choices of the circuit.
The remainder of the paper proceeds as follows. We will first review the classical Fourier framework for the study of interpolating models and develop explicit formulae for the error in these models to produce a basic example of benign overfitting (Sec. 2). We will then construct a quantum model with analogous components to the classical model, and demonstrate how each of these components is related to the structure of the corresponding quantum circuit and measurement (Sec. 3). We then analyze specific cases that give rise to "spikiness" and benign overfitting in these quantum models (Sec. 3.2)."""
# # arXiv:2302.01365v3
# introduction = r"""In order for a learning model to generalise well from training data, it is often crucial to encode some knowledge about the structure of the data into the model itself. Convolutional neural networks are a classic illustration of this principle, whose success at image related tasks is often credited to the existence of model structures that relate to label invariance of the data under translation symmetries. Together with the choice of loss function and hyperparameters, these structures form part of the basic assumptions that a learning model makes about the data, which is commonly referred to as the _inductive bias_ of the model.
# One of the central challenges facing quantum machine learning is to identify data structures that can be encoded usefully into quantum learning models; in other words, what are the forms of inductive bias that naturally lend themselves to quantum computation? In answering this question, we should be wary of hoping for a one-size-fits-all approach in whichquantum models outperform neural network models at generic learning tasks. Rather, effort should be placed in understanding how the Hilbert space structure and probabilistic nature of the theory suggest particular biases for which quantum machine learning may excel. Indeed, an analogous perspective is commonplace in quantum computation, where computational advantages are expected only for specific problems that happen to benefit from the peculiarities of quantum logic.
# In the absence of large quantum computers and in the infancy of quantum machine learning theory, how should we look for insight on this issue? One possibility is to turn to complexity theory, where asymptotic advantages of quantum learning algorithms have been proven. These results are few and far between however, and the enormous gap between what is possible to prove in a complexity-theoretic sense, and the types of advantages that may be possible in practice, means that there are growing doubts about the practical relevance of these results. Indeed, progress in machine learning is often the result of good ideas built on intuition, rather than worst-case complexity theoretic analysis. To repeat a common quip: many problems in machine learning are NP-hard, but neural networks don't know that so they solve them anyway.
# We will take a different route, and lean on the field of quantum foundations to guide us. Quantum foundations is predominantly concerned with understanding the frontier between the quantum and classical world, and typically values a clear qualitative understanding of a phenomenon over purely mathematical knowledge. For these reasons it is well suited to identify features of quantum theory that may advance quantum machine learning in useful directions. In particular, we focus on the phenomenon of contextuality, which is perhaps the most prominent form of nonclassicality studied in the literature. Contextuality has a considerable tradition of being studied in relation to quantum computation, where it is closely connected to the possibility of computational speed-up. Despite this, it has had relatively little attention in quantum machine learning, with only a couple of works linking contextuality to implications for learning.
# We adopt a notion of contextuality called \textit{generalised contextuality}, introduced by Spekkens in 2004. Loosely speaking, it refers to the fact that (i) there are different experimental procedures (called contexts) in the theory that are indistinguishable1, and (ii) any statistical model that reproduces the predictions of the theory must take these contexts into account. With this choice, our first task will then be to introduce a framework to talk about generalised contextuality in machine learning (Section 2). This was missing in previous works, which prove consequences for learning based on phenomena originating from contextuality, but do not attempt to define a notion of contextuality for machine learning that captures a wide range of models. Our general philosophy will be that the framework should depend purely on what a learning model can do, and not on the details of how it does it; i.e., the framework should be independent of the theory on which the models are built. This is necessary to have a framework that treats quantum and classical algorithms on the same footing, and ultimately involves adopting definitions in a similar spirit to the notion of operational contextuality as recently described in.
# We mostly focus on a paradigm of machine learning called multi-task learning, in which the aim is to simultaneously learn a number of separate models for a collection of different (but typically correlated) tasks. Multi-task learning scenarios are conceptually similar to commonly studied contextuality scenarios, and this similarity leads us to a definition of what it means for a multi-task model to be contextual (Section 3). Although the focus on multi-class learning problems appears restrictive, as the separation between tasks is arbitrary at the mathematical level, we also arrive at a notion of contextuality in the single task setting (Section 6). In particular, we argue that it makes sense to think of contextuality as a property relative to a particular inductive bias of a model, rather than a property of the model as a whole.
# Once we have described our framework, our second task will be to identify specific learning problems for which contextuality plays a role (Section 4). We show that this is the case when learning probabilistic models from data sets which feature a linearly conserved quantity in a discrete label space (see Figure 1). Such data sets can arise naturally from experiments involving conserved quantities, zero-sum game scenarios, logistics with conserved resources, substance diffusion in biological systems, and human mobility and migration. We show that the ability of a model to encode the conserved quantity as an inductive bias directly links to a central concept in generalised contextuality, called \textit{operational equivalence}. This results in a constraint on noncontextual learning models that encode the desired bias, which amounts to a limit on the expressivity of noncontextual model classes. For certain data sets, this limitation can negatively impact generalisation performance due to the lack of a suitable model within the class that matches the underlying data distribution; in such cases contextuality may therefore be required for learning. To illustrate this point, in Section 5 we construct a toy problem based on the rock, paper, scissors zero-sum game and prove precise limits on the expressivity of noncontextual model classes that attempt to learn the payoff behaviour of the game.
# In the final part of the work, we study the performance of quantum models for problems that involve our contextuality-inspired bias (Section 7). We first describe two approaches to construct quantum ansatze encoding the bias. The first of these encodes the bias into the state structure of the ansatz, and exploits tools from geometric quantum machine learning. The second approach encodes the bias into the measurement structure, and we present a new family of measurements to this end that may be of independent interest. We then use these tools in a simple numerical investigation (Section 8), inspired by a recent work of Schreiber et al.. Using the fact that quantum machine learning models are equivalent to truncated Fourier series, the authors of define the notion of a classical surrogate model: a linear Fourier features model that has access to the same frequencies of the quantum model, but which lacks its specific inductive bias. The authors found that classical surrogate model classes perform better than quantum model classes on a wide range of regression tasks, the message being that it is still unclear what the inductive bias of quantum machine learning is useful for. In our numerical study, we show that a quantum model class that encodes our contextuality-inspired bias achieves a lower generalisation error than the corresponding surrogate model classes at a specific learning task, even after allowing for regularisation in the surrogate model. We argue that this is due to the fact that the bias cannot be easily encoded into the surrogate model class, which therefore cannot exploit this information during learning.
# In Section 9 we elaborate on a number of areas where contextuality-inspired inductive bias can be expected to play a role in learning. Many of these areas are classical in nature, and therefore suggests that quantum machine learning may be suited to tackling classical learning problems with a specific structure. Finally, in Section 10, we outline our vision for this line of research and the possible next steps to take. Overall, we hope our approach and framework will lead to a new way of thinking about quantum machine learning, and ultimately lead to the identification of impactful problems where the specific structure of quantum theory makes quantum models the machine learning models of choice."""
if __name__ == "__main__":
final_intro, bib_file_content = ez_cite(introduction, debug=True)
|