mrchtr commited on
Commit
10641ee
1 Parent(s): 57009e2

Add initial app version

Browse files
.gitattributes CHANGED
@@ -25,3 +25,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
  *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
28
+ documentstore_german-election-idx.pkl filter=lfs diff=lfs merge=lfs -text
29
+ adapted-retriever/config.json filter=lfs diff=lfs merge=lfs -text
30
+ adapted-retriever/config_sentence_transformers.json filter=lfs diff=lfs merge=lfs -text
31
+ adapted-retriever/modules.json filter=lfs diff=lfs merge=lfs -text
32
+ adapted-retriever/sentence_bert_config.json filter=lfs diff=lfs merge=lfs -text
33
+ adapted-retriever/special_tokens_map.json filter=lfs diff=lfs merge=lfs -text
34
+ adapted-retriever/tokenizer.json filter=lfs diff=lfs merge=lfs -text
35
+ adapted-retriever/tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
adapted-retriever/.gitattributes ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
2
+ sentencepiece.bpe.model filter=lfs diff=lfs merge=lfs -text
adapted-retriever/1_Pooling/config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "word_embedding_dimension": 768,
3
+ "pooling_mode_cls_token": false,
4
+ "pooling_mode_mean_tokens": true,
5
+ "pooling_mode_max_tokens": false,
6
+ "pooling_mode_mean_sqrt_len_tokens": false
7
+ }
adapted-retriever/README.md ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ pipeline_tag: sentence-similarity
3
+ tags:
4
+ - sentence-transformers
5
+ - feature-extraction
6
+ - sentence-similarity
7
+ - transformers
8
+ ---
9
+
10
+ # {MODEL_NAME}
11
+
12
+ This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
13
+
14
+ <!--- Describe your model here -->
15
+
16
+ ## Usage (Sentence-Transformers)
17
+
18
+ Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
19
+
20
+ ```
21
+ pip install -U sentence-transformers
22
+ ```
23
+
24
+ Then you can use the model like this:
25
+
26
+ ```python
27
+ from sentence_transformers import SentenceTransformer
28
+ sentences = ["This is an example sentence", "Each sentence is converted"]
29
+
30
+ model = SentenceTransformer('{MODEL_NAME}')
31
+ embeddings = model.encode(sentences)
32
+ print(embeddings)
33
+ ```
34
+
35
+
36
+
37
+ ## Usage (HuggingFace Transformers)
38
+ Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
39
+
40
+ ```python
41
+ from transformers import AutoTokenizer, AutoModel
42
+ import torch
43
+
44
+
45
+ #Mean Pooling - Take attention mask into account for correct averaging
46
+ def mean_pooling(model_output, attention_mask):
47
+ token_embeddings = model_output[0] #First element of model_output contains all token embeddings
48
+ input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
49
+ return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
50
+
51
+
52
+ # Sentences we want sentence embeddings for
53
+ sentences = ['This is an example sentence', 'Each sentence is converted']
54
+
55
+ # Load model from HuggingFace Hub
56
+ tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
57
+ model = AutoModel.from_pretrained('{MODEL_NAME}')
58
+
59
+ # Tokenize sentences
60
+ encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
61
+
62
+ # Compute token embeddings
63
+ with torch.no_grad():
64
+ model_output = model(**encoded_input)
65
+
66
+ # Perform pooling. In this case, mean pooling.
67
+ sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
68
+
69
+ print("Sentence embeddings:")
70
+ print(sentence_embeddings)
71
+ ```
72
+
73
+
74
+
75
+ ## Evaluation Results
76
+
77
+ <!--- Describe how your model was evaluated -->
78
+
79
+ For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
80
+
81
+
82
+ ## Training
83
+ The model was trained with the parameters:
84
+
85
+ **DataLoader**:
86
+
87
+ `torch.utils.data.dataloader.DataLoader` of length 85 with parameters:
88
+ ```
89
+ {'batch_size': 16, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
90
+ ```
91
+
92
+ **Loss**:
93
+
94
+ `sentence_transformers.losses.MarginMSELoss.MarginMSELoss`
95
+
96
+ Parameters of the fit()-Method:
97
+ ```
98
+ {
99
+ "epochs": 1,
100
+ "evaluation_steps": 0,
101
+ "evaluator": "NoneType",
102
+ "max_grad_norm": 1,
103
+ "optimizer_class": "<class 'transformers.optimization.AdamW'>",
104
+ "optimizer_params": {
105
+ "lr": 2e-05
106
+ },
107
+ "scheduler": "WarmupLinear",
108
+ "steps_per_epoch": null,
109
+ "warmup_steps": 8,
110
+ "weight_decay": 0.01
111
+ }
112
+ ```
113
+
114
+
115
+ ## Full Model Architecture
116
+ ```
117
+ SentenceTransformer(
118
+ (0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: XLMRobertaModel
119
+ (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
120
+ )
121
+ ```
122
+
123
+ ## Citing & Authors
124
+
125
+ <!--- Describe where people can find more information -->
adapted-retriever/config.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29703b29b31e2dabfcd73e52ba0856489249af29f2c8fc5209415fccadfac0d3
3
+ size 821
adapted-retriever/config_sentence_transformers.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8c64b5cece00d8424b4896ea75b512b6008576088497609dfeb6bd63e6d36b8
3
+ size 122
adapted-retriever/modules.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f4b264b80206c830bebbdcae377e137925650a433b689343a63bdc9b3145460
3
+ size 229
adapted-retriever/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73203f5f04e88a22c1a336c4aceb89220dd8b1589151be83576767f781d2c00c
3
+ size 1112244081
adapted-retriever/sentence_bert_config.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec8e29d6dcb61b611b7d3fdd2982c4524e6ad985959fa7194eacfb655a8d0d51
3
+ size 53
adapted-retriever/sentencepiece.bpe.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
3
+ size 5069051
adapted-retriever/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:378eb3bf733eb16e65792d7e3fda5b8a4631387ca04d2015199c4d4f22ae554d
3
+ size 239
adapted-retriever/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46afe88da5fd71bdbab5cfab5e84c1adce59c246ea5f9341bbecef061891d0a7
3
+ size 17082913
adapted-retriever/tokenizer_config.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c84cba673d65cd6fabcaf0340ae8e57b34306e01862132f4b476936917727dea
3
+ size 483
app.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ # My first app
3
+ Here's our first attempt at using data to create a table:
4
+ """
5
+ import streamlit as st
6
+ import pandas as pd
7
+ from load_css import local_css
8
+ from retriever import do_search
9
+
10
+ local_css('style.css')
11
+
12
+ st.header('Semantic search demo')
13
+ search = st.text_input('')
14
+
15
+ if search:
16
+ result = do_search(search)
17
+ col1, col2, col3 = st.columns(3)
18
+
19
+ with col1:
20
+ st.write('TF-IDF')
21
+ st.write(result[0])
22
+
23
+ with col2:
24
+ st.write('Base dense retriever')
25
+ st.write(result[1])
26
+
27
+ with col3:
28
+ st.write('Adapted dense retriever')
29
+ st.write(result[2])
30
+
31
+
32
+
documentstore_german-election-idx.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc6e85c8a51b19f5b37df691bcfc75b57b1d24086b4e004489964a45927f9024
3
+ size 4777552
requirements.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ farm-haystack
retriever.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from haystack.document_stores import InMemoryDocumentStore
2
+ from haystack.utils import convert_files_to_docs
3
+ from haystack.nodes.retriever import TfidfRetriever
4
+ from haystack.pipelines import DocumentSearchPipeline, ExtractiveQAPipeline
5
+ from haystack.nodes.retriever import EmbeddingRetriever
6
+ from haystack.nodes import FARMReader
7
+ import pickle
8
+ from pprint import pprint
9
+
10
+ class ExportableInMemoryDocumentStore(InMemoryDocumentStore):
11
+ """
12
+ Wrapper class around the InMemoryDocumentStore.
13
+ When the application is deployed to Huggingface Spaces there will be no GPU available.
14
+ We need to load pre-calculated data into the InMemoryDocumentStore.
15
+ """
16
+ def export(self, file_name='in_memory_store.pkl'):
17
+ with open(file_name, 'wb') as f:
18
+ pickle.dump(self.indexes, f)
19
+
20
+ def load_data(self, file_name='in_memory_store.pkl'):
21
+ with open(file_name, 'rb') as f:
22
+ self.indexes = pickle.load(f)
23
+
24
+
25
+
26
+ document_store = ExportableInMemoryDocumentStore(similarity='cosine')
27
+ document_store.load_data('documentstore_german-election-idx.pkl')
28
+
29
+ retriever = TfidfRetriever(document_store=document_store)
30
+ base_dense_retriever = EmbeddingRetriever(
31
+ document_store=document_store,
32
+ embedding_model='sentence-transformers/paraphrase-multilingual-mpnet-base-v2',
33
+ model_format='sentence_transformers'
34
+ )
35
+
36
+ fine_tuned_retriever = EmbeddingRetriever(
37
+ document_store=document_store,
38
+ embedding_model='./adapted-retriever',
39
+ model_format='sentence_transformers'
40
+ )
41
+
42
+ def sparse_retrieval(query):
43
+ """Sparse retrieval pipeline"""
44
+ p_retrieval = DocumentSearchPipeline(retriever)
45
+ return p_retrieval.run(query=query)
46
+
47
+ def dense_retrieval(query, retriever='base'):
48
+ if retriever == 'base':
49
+ p_retrieval = DocumentSearchPipeline(base_dense_retriever)
50
+ elif retriever == 'adapted':
51
+ p_retrieval = DocumentSearchPipeline(fine_tuned_retriever)
52
+ else:
53
+ return None
54
+
55
+ return p_retrieval.run(query=query)
56
+
57
+
58
+ def do_search(query):
59
+ sparse_result = sparse_retrieval(query)['documents'][0].content
60
+ dense_base_result = dense_retrieval(query, retriever='base')['documents'][0].content
61
+ dense_adapted_result = dense_retrieval(query, retriever='adapted')['documents'][0].content
62
+ return sparse_result, dense_base_result, dense_adapted_result
63
+
64
+ if __name__ == '__main__':
65
+ query = 'Klimawandel stoppen?'
66
+ result = do_search(query)
67
+ pprint(result)
68
+
69
+