Repository: microsoft/promptbench
Issue: 执行bertattack攻击的时候出现SSL错误，是不是下载bert模型出错啊！基本LLM模型已经是在本地了
Root cause: Environment Incompatibility

---------------------------------------------------------------------------
SSLCertVerificationError                  Traceback (most recent call last)
File ~/miniconda3/envs/promptbench/lib/python3.9/urllib/request.py:1346, in AbstractHTTPHandler.do_open(self, http_class, req, **http_conn_args)
   1345 try:
-> 1346     h.request(req.get_method(), req.selector, req.data, headers,
   1347               encode_chunked=req.has_header('Transfer-encoding'))
   1348 except OSError as err: # timeout error

File ~/miniconda3/envs/promptbench/lib/python3.9/http/client.py:1285, in HTTPConnection.request(self, method, url, body, headers, encode_chunked)
   1284 """Send a complete request to the server."""
-> 1285 self._send_request(method, url, body, headers, encode_chunked)

File ~/miniconda3/envs/promptbench/lib/python3.9/http/client.py:1331, in HTTPConnection._send_request(self, method, url, body, headers, encode_chunked)
   1330     body = _encode(body, 'body')
-> 1331 self.endheaders(body, encode_chunked=encode_chunked)

File ~/miniconda3/envs/promptbench/lib/python3.9/http/client.py:1280, in HTTPConnection.endheaders(self, message_body, encode_chunked)
   1279     raise CannotSendHeader()
-> 1280 self._send_output(message_body, encode_chunked=encode_chunked)

File ~/miniconda3/envs/promptbench/lib/python3.9/http/client.py:1040, in HTTPConnection._send_output(self, message_body, encode_chunked)
   1039 del self._buffer[:]
-> 1040 self.send(msg)
   1042 if message_body is not None:
   1043 
   1044     # create a consistent interface to message_body

File ~/miniconda3/envs/promptbench/lib/python3.9/http/client.py:980, in HTTPConnection.send(self, data)
    979 if self.auto_open:
--> 980     self.connect()
    981 else:

File ~/miniconda3/envs/promptbench/lib/python3.9/http/client.py:1454, in HTTPSConnection.connect(self)
   1452     server_hostname = self.host
-> 1454 self.sock = self._context.wrap_socket(self.sock,
   1455                                       server_hostname=server_hostname)

File ~/miniconda3/envs/promptbench/lib/python3.9/ssl.py:501, in SSLContext.wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
    495 def wrap_socket(self, sock, server_side=False,
    496                 do_handshake_on_connect=True,
    497                 suppress_ragged_eofs=True,
    498                 server_hostname=None, session=None):
    499     # SSLSocket class handles server_hostname encoding before it calls
    500     # ctx._wrap_socket()
--> 501     return self.sslsocket_class._create(
    502         sock=sock,
    503         server_side=server_side,
    504         do_handshake_on_connect=do_handshake_on_connect,
    505         suppress_ragged_eofs=suppress_ragged_eofs,
    506         server_hostname=server_hostname,
    507         context=self,
    508         session=session
    509     )

File ~/miniconda3/envs/promptbench/lib/python3.9/ssl.py:1074, in SSLSocket._create(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)
   1073             raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-> 1074         self.do_handshake()
   1075 except (OSError, ValueError):

File ~/miniconda3/envs/promptbench/lib/python3.9/ssl.py:1343, in SSLSocket.do_handshake(self, block)
   1342         self.settimeout(None)
-> 1343     self._sslobj.do_handshake()
   1344 finally:

SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1133)

During handling of the above exception, another exception occurred:

URLError                                  Traceback (most recent call last)
Cell In[17], line 5
      3 attack = Attack(model_t5, "bertattack", train_dataset, prompt, eval_func, unmodifiable_words, verbose=True)
      4 # print attack result
----> 5 print(attack.attack())

File ~/autodl-tmp/promptbench/promptbench/prompt_attack/attack.py:234, in Attack.attack(self)
    231     return results
    233 else:
--> 234     return self.prompt_attack.attack(self.prompt)

File ~/autodl-tmp/promptbench/promptbench/prompt_attack/attack.py:660, in AdvPromptAttack.attack(self, example)
    658     return SkippedAttackResult(goal_function_result)
    659 else:
--> 660     result = self._attack(goal_function_result)
    661     return result

File ~/autodl-tmp/promptbench/promptbench/prompt_attack/attack.py:602, in AdvPromptAttack._attack(self, initial_result)
    591 def _attack(self, initial_result):
    592     """Calls the ``SearchMethod`` to perturb the ``AttackedText`` stored in
    593     ``initial_result``.
    594 
   (...)
    600             or ``MaximizedAttackResult``.
    601     """
--> 602     final_result = self.search_method(initial_result)
    603     self.clear_cache()
    604     if final_result.goal_status == GoalFunctionResultStatus.SUCCEEDED:

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/textattack/search_methods/search_method.py:35, in SearchMethod.__call__(self, initial_result)
     30 if not hasattr(self, "filter_transformations"):
     31     raise AttributeError(
     32         "Search Method must have access to filter_transformations method"
     33     )
---> 35 result = self.perform_search(initial_result)
     36 # ensure that the number of queries for this GoalFunctionResult is up-to-date
     37 result.num_queries = self.goal_function.num_queries

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/textattack/search_methods/greedy_word_swap_wir.py:141, in GreedyWordSwapWIR.perform_search(self, initial_result)
    139 results = None
    140 while i < len(index_order) and not search_over:
--> 141     transformed_text_candidates = self.get_transformations(
    142         cur_result.attacked_text,
    143         original_text=initial_result.attacked_text,
    144         indices_to_modify=[index_order[i]],
    145     )
    146     i += 1
    147     if len(transformed_text_candidates) == 0:

File ~/autodl-tmp/promptbench/promptbench/prompt_attack/attack.py:519, in AdvPromptAttack.get_transformations(self, current_text, original_text, **kwargs)
    514 else:
    515     transformed_texts = self._get_transformations_uncached(
    516         current_text, original_text, **kwargs
    517     )
--> 519 return self.filter_transformations(
    520     transformed_texts, current_text, original_text
    521 )

File ~/autodl-tmp/promptbench/promptbench/prompt_attack/attack.py:584, in AdvPromptAttack.filter_transformations(self, transformed_texts, current_text, original_text)
    582         if self.constraints_cache[(current_text, transformed_text)]:
    583             filtered_texts.append(transformed_text)
--> 584 filtered_texts += self._filter_transformations_uncached(
    585     uncached_texts, current_text, original_text=original_text
    586 )
    587 # Sort transformations to ensure order is preserved between runs
    588 filtered_texts.sort(key=lambda t: t.text)

File ~/autodl-tmp/promptbench/promptbench/prompt_attack/attack.py:544, in AdvPromptAttack._filter_transformations_uncached(self, transformed_texts, current_text, original_text)
    539     if not original_text:
    540         raise ValueError(
    541             f"Missing `original_text` argument when constraint {type(C)} is set to compare against `original_text`"
    542         )
--> 544     filtered_texts = C.call_many(filtered_texts, original_text)
    545 else:
    546     filtered_texts = C.call_many(filtered_texts, current_text)

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/textattack/constraints/constraint.py:50, in Constraint.call_many(self, transformed_texts, reference_text)
     46     except KeyError:
     47         raise KeyError(
     48             "transformed_text must have `last_transformation` attack_attr to apply constraint"
     49         )
---> 50 filtered_texts = self._check_constraint_many(
     51     compatible_transformed_texts, reference_text
     52 )
     53 return list(filtered_texts) + incompatible_transformed_texts

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/textattack/constraints/semantics/sentence_encoders/sentence_encoder.py:179, in SentenceEncoder._check_constraint_many(self, transformed_texts, reference_text)
    175 def _check_constraint_many(self, transformed_texts, reference_text):
    176     """Filters the list ``transformed_texts`` so that the similarity
    177     between the ``reference_text`` and the transformed text is greater than
    178     the ``self.threshold``."""
--> 179     scores = self._score_list(reference_text, transformed_texts)
    181     for i, transformed_text in enumerate(transformed_texts):
    182         # Optionally ignore similarity score for sentences shorter than the
    183         # window size.
    184         if (
    185             self.skip_text_shorter_than_window
    186             and len(transformed_text.words) < self.window_size
    187         ):

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/textattack/constraints/semantics/sentence_encoders/sentence_encoder.py:152, in SentenceEncoder._score_list(self, starting_text, transformed_texts)
    142     starting_text_windows.append(
    143         starting_text.text_window_around_index(
    144             modified_index, self.window_size
    145         )
    146     )
    147     transformed_text_windows.append(
    148         transformed_text.text_window_around_index(
    149             modified_index, self.window_size
    150         )
    151     )
--> 152 embeddings = self.encode(starting_text_windows + transformed_text_windows)
    153 if not isinstance(embeddings, torch.Tensor):
    154     embeddings = torch.tensor(embeddings)

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/textattack/constraints/semantics/sentence_encoders/universal_sentence_encoder/universal_sentence_encoder.py:30, in UniversalSentenceEncoder.encode(self, sentences)
     28 def encode(self, sentences):
     29     if not self.model:
---> 30         self.model = hub.load(self._tfhub_url)
     31     encoding = self.model(sentences)
     33     if isinstance(encoding, dict):

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/tensorflow_hub/module_v2.py:100, in load(handle, tags, options)
     98 if not isinstance(handle, str):
     99   raise ValueError("Expected a string, got %s" % handle)
--> 100 module_path = resolve(handle)
    101 is_hub_module_v1 = tf.io.gfile.exists(_get_module_proto_path(module_path))
    102 if tags is None and is_hub_module_v1:

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/tensorflow_hub/module_v2.py:55, in resolve(handle)
     31 def resolve(handle):
     32   """Resolves a module handle into a path.
     33 
     34   This function works both for plain TF2 SavedModels and the legacy TF1 Hub
   (...)
     53     A string representing the Module path.
     54   """
---> 55   return registry.resolver(handle)

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/tensorflow_hub/registry.py:49, in MultiImplRegister.__call__(self, *args, **kwargs)
     47 for impl in reversed(self._impls):
     48   if impl.is_supported(*args, **kwargs):
---> 49     return impl(*args, **kwargs)
     50   else:
     51     fails.append(type(impl).__name__)

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/tensorflow_hub/compressed_module_resolver.py:81, in HttpCompressedFileResolver.__call__(self, handle)
     77   response = self._call_urlopen(request)
     78   return resolver.DownloadManager(handle).download_and_uncompress(
     79       response, tmp_dir)
---> 81 return resolver.atomic_download(handle, download, module_dir,
     82                                 self._lock_file_timeout_sec())

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/tensorflow_hub/resolver.py:421, in atomic_download(handle, download_fn, module_dir, lock_file_timeout_sec)
    419 logging.info("Downloading TF-Hub Module '%s'.", handle)
    420 tf.compat.v1.gfile.MakeDirs(tmp_dir)
--> 421 download_fn(handle, tmp_dir)
    422 # Write module descriptor to capture information about which module was
    423 # downloaded by whom and when. The file stored at the same level as a
    424 # directory in order to keep the content of the 'model_dir' exactly as it
   (...)
    429 # module caching protocol and no code in the TF-Hub library reads its
    430 # content.
    431 _write_module_descriptor_file(handle, module_dir)

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/tensorflow_hub/compressed_module_resolver.py:77, in HttpCompressedFileResolver.__call__.<locals>.download(handle, tmp_dir)
     71   return resolver.DownloadManager(handle).download_and_uncompress(
     72       response, tmp_dir
     73   )
     75 request = urllib.request.Request(
     76     self._append_compressed_format_query(handle))
---> 77 response = self._call_urlopen(request)
     78 return resolver.DownloadManager(handle).download_and_uncompress(
     79     response, tmp_dir)

File ~/miniconda3/envs/promptbench/lib/python3.9/site-packages/tensorflow_hub/resolver.py:528, in HttpResolverBase._call_urlopen(self, request)
    526   return urllib.request.urlopen(request)
    527 else:
--> 528   return urllib.request.urlopen(request, context=self._context)

File ~/miniconda3/envs/promptbench/lib/python3.9/urllib/request.py:214, in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    212 else:
    213     opener = _opener
--> 214 return opener.open(url, data, timeout)

File ~/miniconda3/envs/promptbench/lib/python3.9/urllib/request.py:517, in OpenerDirector.open(self, fullurl, data, timeout)
    514     req = meth(req)
    516 sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
--> 517 response = self._open(req, data)
    519 # post-process response
    520 meth_name = protocol+"_response"

File ~/miniconda3/envs/promptbench/lib/python3.9/urllib/request.py:534, in OpenerDirector._open(self, req, data)
    531     return result
    533 protocol = req.type
--> 534 result = self._call_chain(self.handle_open, protocol, protocol +
    535                           '_open', req)
    536 if result:
    537     return result

File ~/miniconda3/envs/promptbench/lib/python3.9/urllib/request.py:494, in OpenerDirector._call_chain(self, chain, kind, meth_name, *args)
    492 for handler in handlers:
    493     func = getattr(handler, meth_name)
--> 494     result = func(*args)
    495     if result is not None:
    496         return result

File ~/miniconda3/envs/promptbench/lib/python3.9/urllib/request.py:1389, in HTTPSHandler.https_open(self, req)
   1388 def https_open(self, req):
-> 1389     return self.do_open(http.client.HTTPSConnection, req,
   1390         context=self._context, check_hostname=self._check_hostname)

File ~/miniconda3/envs/promptbench/lib/python3.9/urllib/request.py:1349, in AbstractHTTPHandler.do_open(self, http_class, req, **http_conn_args)
   1346         h.request(req.get_method(), req.selector, req.data, headers,
   1347                   encode_chunked=req.has_header('Transfer-encoding'))
   1348     except OSError as err: # timeout error
-> 1349         raise URLError(err)
   1350     r = h.getresponse()
   1351 except:

URLError: <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1133)>

您好，看起来应该是因为是下载sentence encoder模型出错：textattack库在amazon的aws下载该模型，解决方案是可以先科学上网下载到本地，然后修改textattacks库里对应的load代码。

衷心感谢 我马上修改然后尝试一下

我已经找到了load代码
    if large:
            tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder-large/5"
        else:
            tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder/3"

        self._tfhub_url = tfhub_url
 但是我不清楚后面的数字代表的是什么模型版本吗？

> 我已经找到了load代码 if large: tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder-large/5" else: tfhub_url = "https://tfhub.dev/google/universal-sentence-encoder/3"
> 
> ```
>     self._tfhub_url = tfhub_url
> ```
> 
> 但是我不清楚后面的数字代表的是什么模型版本吗？

是的，两个都下载到本地然后替换一下路径应该就可以了！

好的谢谢啦

Stale issue message