zstanjj
/

HTML-Pruner-Phi-3.8B

@@ -48,7 +48,7 @@ question = "When was the bellagio in las vegas built?"
 html = """
 <html>
 <head>
-<title>When was the bellagio in las vegas built?</title>
 </head>
 <body>
 <p class="class0">The Bellagio is a luxury hotel and casino located on the Las Vegas Strip in Paradise, Nevada. It was built in 1998.</p>
@@ -76,7 +76,7 @@ simplified_html = clean_html(html)
 print(simplified_html)
 # <html>
-# <title>When was the bellagio in las vegas built?</title>
 # <p>The Bellagio is a luxury hotel and casino located on the Las Vegas Strip in Paradise, Nevada. It was built in 1998.</p>
 # <div>
 # <p>Some other text</p>
@@ -111,14 +111,14 @@ MAX_CONTEXT_WINDOW_GEN = 32
 from htmlrag import build_block_tree
 block_tree, simplified_html = build_block_tree(simplified_html, max_node_words=MAX_NODE_WORDS_EMBED)
-# block_tree, simplified_html=build_block_tree(simplified_html, max_node_words=MAX_NODE_WORDS_GEN, zh_char=True) # for Chinese text
 for block in block_tree:
     print("Block Content: ", block[0])
     print("Block Path: ", block[1])
     print("Is Leaf: ", block[2])
     print("")
-# Block Content:  <title>When was the bellagio in las vegas built?</title>
 # Block Path:  ['html', 'title']
 # Is Leaf:  True
 #
@@ -139,7 +139,7 @@ for block in block_tree:
 ```python
 from htmlrag import EmbedHTMLPruner
-embed_model="/train_data_load/huggingface/tjj_hf/bge-large-en/"
 query_instruction_for_retrieval = "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: "
 embed_html_pruner = EmbedHTMLPruner(embed_model=embed_model, local_inference=True, query_instruction_for_retrieval = query_instruction_for_retrieval)
 # alternatively you can init a remote TEI model, refer to https://github.com/huggingface/text-embeddings-inference.
@@ -148,15 +148,15 @@ embed_html_pruner = EmbedHTMLPruner(embed_model=embed_model, local_inference=Tru
 block_rankings=embed_html_pruner.calculate_block_rankings(question, simplified_html, block_tree)
 print(block_rankings)
-# [0, 2, 1]
 #. alternatively you can use bm25 to rank the blocks
 from htmlrag import BM25HTMLPruner
 bm25_html_pruner = BM25HTMLPruner()
-block_rankings=bm25_html_pruner.calculate_block_rankings(question, simplified_html, block_tree)
 print(block_rankings)
-# [0, 2, 1]
 from transformers import AutoTokenizer
@@ -166,7 +166,7 @@ pruned_html = embed_html_pruner.prune_HTML(simplified_html, block_tree, block_ra
 print(pruned_html)
 # <html>
-# <title>When was the bellagio in las vegas built?</title>
 # <p>The Bellagio is a luxury hotel and casino located on the Las Vegas Strip in Paradise, Nevada. It was built in 1998.</p>
 # </html>
 ```
@@ -179,15 +179,15 @@ from htmlrag import GenHTMLPruner
 import torch
 # construct a finer block tree
-block_tree, pruned_html=build_block_tree(pruned_html, max_node_words=MAX_NODE_WORDS_GEN)
-# block_tree, pruned_html=build_block_tree(pruned_html, max_node_words=MAX_NODE_WORDS_GEN, zh_char=True) # for Chinese text
 for block in block_tree:
     print("Block Content: ", block[0])
     print("Block Path: ", block[1])
     print("Is Leaf: ", block[2])
     print("")
-# Block Content:  <title>When was the bellagio in las vegas built?</title>
 # Block Path:  ['html', 'title']
 # Is Leaf:  True
 #
@@ -200,8 +200,8 @@ if torch.cuda.is_available():
     device="cuda"
 else:
     device="cpu"
-gen_embed_pruner = GenHTMLPruner(gen_model=ckpt_path, max_node_words=MAX_NODE_WORDS_GEN, device=device)
-block_rankings=gen_embed_pruner.calculate_block_rankings(question, pruned_html, block_tree)
 print(block_rankings)
 # [1, 0]

 html = """
 <html>
 <head>
+<h1>Bellagio Hotel in Las</h1>
 </head>
 <body>
 <p class="class0">The Bellagio is a luxury hotel and casino located on the Las Vegas Strip in Paradise, Nevada. It was built in 1998.</p>
 print(simplified_html)
 # <html>
+# <h1>Bellagio Hotel in Las</h1>
 # <p>The Bellagio is a luxury hotel and casino located on the Las Vegas Strip in Paradise, Nevada. It was built in 1998.</p>
 # <div>
 # <p>Some other text</p>
 from htmlrag import build_block_tree
 block_tree, simplified_html = build_block_tree(simplified_html, max_node_words=MAX_NODE_WORDS_EMBED)
+# block_tree, simplified_html = build_block_tree(simplified_html, max_node_words=MAX_NODE_WORDS_GEN, zh_char=True) # for Chinese text
 for block in block_tree:
     print("Block Content: ", block[0])
     print("Block Path: ", block[1])
     print("Is Leaf: ", block[2])
     print("")
+# Block Content:  <h1>Bellagio Hotel in Las</h1>
 # Block Path:  ['html', 'title']
 # Is Leaf:  True
 #
 ```python
 from htmlrag import EmbedHTMLPruner
+embed_model="BAAI/bge-large-en"
 query_instruction_for_retrieval = "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: "
 embed_html_pruner = EmbedHTMLPruner(embed_model=embed_model, local_inference=True, query_instruction_for_retrieval = query_instruction_for_retrieval)
 # alternatively you can init a remote TEI model, refer to https://github.com/huggingface/text-embeddings-inference.
 block_rankings=embed_html_pruner.calculate_block_rankings(question, simplified_html, block_tree)
 print(block_rankings)
+# [2, 0, 1]
 #. alternatively you can use bm25 to rank the blocks
 from htmlrag import BM25HTMLPruner
 bm25_html_pruner = BM25HTMLPruner()
+block_rankings = bm25_html_pruner.calculate_block_rankings(question, simplified_html, block_tree)
 print(block_rankings)
+# [2, 0, 1]
 from transformers import AutoTokenizer
 print(pruned_html)
 # <html>
+# <h1>Bellagio Hotel in Las</h1>
 # <p>The Bellagio is a luxury hotel and casino located on the Las Vegas Strip in Paradise, Nevada. It was built in 1998.</p>
 # </html>
 ```
 import torch
 # construct a finer block tree
+block_tree, pruned_html = build_block_tree(pruned_html, max_node_words=MAX_NODE_WORDS_GEN)
+# block_tree, pruned_html = build_block_tree(pruned_html, max_node_words=MAX_NODE_WORDS_GEN, zh_char=True) # for Chinese text
 for block in block_tree:
     print("Block Content: ", block[0])
     print("Block Path: ", block[1])
     print("Is Leaf: ", block[2])
     print("")
+# Block Content:  <h1>Bellagio Hotel in Las</h1>
 # Block Path:  ['html', 'title']
 # Is Leaf:  True
 #
     device="cuda"
 else:
     device="cpu"
+gen_embed_pruner = GenHTMLPruner(gen_model=ckpt_path, device=device)
+block_rankings = gen_embed_pruner.calculate_block_rankings(question, pruned_html, block_tree)
 print(block_rankings)
 # [1, 0]