Repository: knodle/knodle (language: python)

Issue:
Feature/232 refactor data



Discussion / hints:

Fix patch:
diff --git a/.flake8 b/.flake8
new file mode 100644
index 00000000..008010a1
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,5 @@
+[flake8]
+ignore = E203, E266, E501, W503, F403, F401
+max-line-length = 100
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
\ No newline at end of file
diff --git a/.github/workflows/LFS_warning.yml b/.github/workflows/LFS_warning.yml
new file mode 100644
index 00000000..8bab972d
--- /dev/null
+++ b/.github/workflows/LFS_warning.yml
@@ -0,0 +1,13 @@
+name: LFS-warning
+
+on: [ pull_request ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: LFS-warning
+        uses: ActionsDesk/lfs-warning@v2.0
+        with:
+          filesizelimit: '10485760' # 10 MB is 10485760 Bytes
\ No newline at end of file
diff --git a/.github/workflows/all_commits.yml b/.github/workflows/all_commits.yml
index 6676890a..741bca78 100644
--- a/.github/workflows/all_commits.yml
+++ b/.github/workflows/all_commits.yml
@@ -1,6 +1,6 @@
-name: Lint, test on every commit
+name: test
 
-on: [push]
+on: [ push ]
 
 jobs:
   build:
@@ -8,20 +8,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.7
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.7
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements_dev.txt
+          pip install -r requirements.txt
       - name: Lint with flake8
         run: |
+          pip install flake8
           flake8 . --count --select=E4,E7,E9,F4,F63,F7,F82,F401,F901 --show-source --max-line-length=127 --statistics
       - name: Test with pytest
         run: |
-          pytest --cov-config=./tests/.coveragerc --cov --cov-fail-under=60
+          pip install pytest
+          pytest
       - name: Build
         run: |
+          pip install setuptools
           python setup.py sdist bdist_wheel
diff --git a/.github/workflows/main_deploy.yml b/.github/workflows/main_deploy.yml
index fd907eb2..5b8e2966 100644
--- a/.github/workflows/main_deploy.yml
+++ b/.github/workflows/main_deploy.yml
@@ -1,20 +1,33 @@
-name: Build and Deploy on Master
+name: Upload to PIP on Release
 
 on:
-  push:
-    branches:
-      - master
+  release:
+    types: [created]
 
 jobs:
-  build:
+  upload:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Build and deploy
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: "Installs dependencies"
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install setuptools wheel twine
+
+      - name: "Tag Update"
+        run: |
+          python3 update_version_from_tag.py $(git tag --points-at HEAD)
+
+      - name: "Builds and uploads to PyPI"
         run: |
-          pip install -r requirements_dev.txt
-          python setup.py sdist bdist_wheel
-          # twine upload dist/* FOR UPLOADING PURPOSES
+          python3 setup.py sdist bdist_wheel
+          python3 -m twine upload dist/*
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.gitignore b/.gitignore
index 7a249b11..8e5cfefb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 *.so
 
 # Packages
+PKG-INFO
 *.egg
 *.egg-info
 dist
@@ -19,6 +20,9 @@ lib
 lib64
 __pycache__
 
+# package helper
+knodle-*
+
 # Installer logs
 pip-log.txt
 
@@ -35,4 +39,24 @@ nosetests.xml
 .project
 .pydevproject
 
+*.ipynb_checkpoints
+
 /venv
+
+.python-version
+.DS_Store
+
+*.npy
+runs/
+runs_old/
+wandb/
+*.pt
+launch.json
+*.env
+
+# Data:
+
+# Large datasets
+*.lib
+*.csv
+data_from_minio/
\ No newline at end of file
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 00000000..9179e614
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,10 @@
+repos:
+    - repo: https://github.com/ambv/black
+      rev: stable
+      hooks:
+          - id: black
+            language_version: python3.7
+    - repo: https://gitlab.com/pycqa/flake8
+      rev: 3.7.9
+      hooks:
+          - id: flake8
diff --git a/.toml b/.toml
new file mode 100644
index 00000000..02d4b0ca
--- /dev/null
+++ b/.toml
@@ -0,0 +1,16 @@
+[tool.black]
+line-length = 100
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
index 6f93e430..261eeb9e 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,1 +1,201 @@
-TBC
\ No newline at end of file
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 00000000..d8afddc5
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,32 @@
+#
+# Use .add_data_files and .add_data_dir methods in a appropriate
+# setup.py files to include non-python files such as documentation,
+# data, etc files to distribution (*for installation*).
+# Avoid using MANIFEST.in for that.
+#
+include MANIFEST.in
+include pyproject.toml
+include README.md
+include LICENSE
+
+# Note that sub-directories that don't have __init__ are apparently not
+# included by 'recursive-include', so list those separately
+recursive-include knodle *
+recursive-include tests  *
+prune tutorials
+
+# Exclude generated files
+prune */__pycache__
+global-exclude *.pyc *.pyo *.pyd *.swp *.bak *~
+
+#### save for later ####
+# Exclude license file that we append to the main license when running
+# `python setup.py sdist`
+#exclude LICENSES_bundled.txt
+#include pytest.ini
+#include *.txt
+#include site.cfg.example
+#include runtests.py
+#include tox.ini
+#include .coveragerc
+
diff --git a/README.md b/README.md
index bd8ec9fb..fb10a9c3 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,190 @@
-# knodle
+<img src="http://knodle.cc/static/logo-866f6c91f8135d66107abccbec7bf77e.png" height="150">
 
-Knowledge infused deep learning framework
+[![Python Version](https://img.shields.io/badge/python-3.7-yellow.svg)](https://www.python.org/downloads/release/python-360/)
+[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+![PyPI](https://img.shields.io/pypi/v/knodle)
+[![GitHub Release](https://img.shields.io/github/release/knodle/knodle.svg?style=flat)]()  
+[![build status](https://github.com/knodle/knodle/workflows/lint_n_test/badge.svg)](https://github.com/knodle/knodle/actions?branch=style_guide)
 
-## Usage
+Knodle (_Knowledge infused deep learning framework_) provides a modularization for separating weak data annotations, powerful deep learning models, and methods for improving weakly supervised training.
+
+More details about Knodle are in our recent [paper](https://arxiv.org/abs/2104.11557). 
+
+****
+### Latest news
+- **Apr 2021: Knodle first release! :rocket:**
+- **Apr 2021:** Anastasiia Sedova, Andreas Stephan, Marina Speranskaya, Benjamin Roth. [Knodle: Modular Weakly Supervised Learning with PyTorch](https://arxiv.org/abs/2104.11557) (preprint).
 
 ## Installation
 
-## Requirements
+```
+pip install knodle
+```
+
+## Usage
+
+knodle offers various methods for denoising weak supervision sources and improve them. There are several methods available for denoising. Examples can be seen in the tutorials folder.
+
+There are four mandatory inputs for knodle:
+
+1. `model_input_x`: Your model features (e.g. TF-IDF values) without any labels. Shape: (n_instances x features)
+2. `mapping_rules_labels_t`: This matrix maps all weak rules to a label. Shape: (n_rules x n_classes)
+3. `rule_matches_z`: This matrix shows all applied rules on your dataset. Shape: (n_instances x n_rules)
+4. `model`: A PyTorch model which can take your provided `model_input_x` as input. Examples are in the [model folder](https://github.com/knodle/knodle/tree/develop/knodle/model/).
+
+If you know which denoising method you want to use, you can directly call the corresponding module (the list of currently supported methods is provided [below](https://github.com/knodle/knodle/tree/style_guide#denoising-methods)).
+
+Example for training the baseline classifier:
+
+```python
+from knodle.model.logistic_regression_model import LogisticRegressionModel
+from knodle.trainer.baseline.majority import MajorityVoteTrainer
+
+NUM_OUTPUT_CLASSES = 2
+
+model = LogisticRegressionModel(model_input_x.shape[1], NUM_OUTPUT_CLASSES)
+
+trainer = MajorityVoteTrainer(
+  model=model,
+  mapping_rules_labels_t=mapping_rules_labels_t,
+  model_input_x=model_input_x,
+  rule_matches_z=rule_matches_z,
+  dev_model_input_x=X_dev,
+  dev_gold_labels_y=Y_dev
+)
+
+trainer.train()
+
+trainer.test(X_test, Y_test)
+```
+
+A more detailed example of classifier training is [here](https://github.com/knodle/knodle/blob/develop/examples/trainer/simple_auto_trainer/auto_trainer_tutorial.ipynb).  
+
+## Main Principles
+
+The framework provides a simple tensor-driven abstraction based on PyTorch allowing researchers to efficiently develop and compare their methods. The emergence of machine learning software frameworks is the biggest enabler for the wide spread adoption of machine learning and its speed of development. With Knodle we want to empower researchers in a similar fashion.
+
+Knodle main goals:
+- *Data abstraction*. The interface is a tensor-driven data abstraction which unifies a large number of input
+variants and is applicable to a large number of tasks.
+- *Method independence*. We distinguish between weak supervision and prediction model. This enables comparability and accounts for a domain-specific inductive biases.
+- *Accessibility*. There is a high-level access to the library, that makes it easy to test existing methods, incorporate new ones and benchmark them against each other.
+
+
+<img src="img/schema.png" height="300"/>
+
+Apart from that, Knodle includes a selection of well-known data sets from prior work in weak supervision. Knodle ecosystem provides modular access to datasets and denoising methods (that can, in turn, be combined with arbitrary deep learning models), enabling easy experimentation.
+
+Datasets currently provided in Knodle:
+
+- Spam Dataset - a dataset, based on the YouTube comments dataset from [Alberto et al. (2015)](https://www.researchgate.net/publication/300414679_TubeSpam_Comment_Spam_Filtering_on_YouTube). Here, the task is to classify whether a text is relevant to the video or holds spam, such as adver- tisement.
+- Spouse Dataset - relation extraction dataset is based on the Signal Media One-Million News Articles Dataset from [Corney et al. (2016)](http://ceur-ws.org/Vol-1568/paper8.pdf). 
+- IMDb Dataset - a dataset, that consists of short movie reviews. The task is to determine whether a review holds a positive or negative sentiment. 
+- TAC-based Relation Extraction Dataset - a dataset built over Knowledge Base Population challenges in the Text Analysis Conference. For development and test purposes the corpus annotated via crowdsourcing and human labeling from KBP is used ([Zhang et al. (2017)](https://www.aclweb.org/anthology/D17-1004.pdf). The training is done on a weakly-supervised noisy dataset based on TAC KBP corpora ([Surdeanu (2013)](https://tac.nist.gov/publications/2013/additional.papers/KBP2013_English_and_Temporal_Slot_Filling_overview.TAC2013.proceedings.pdf)). 
+
+All datasets are added to the Knodle framework in the tensor format described above and could be dowloaded [here](http://knodle.dm.univie.ac.at/minio/knodle/datasets/).
+To see how the datasets were created please have a look at the [dedicated tutorial](https://github.com/knodle/knodle/tree/develop/examples/data_preprocessing).
+
+
+## Denoising Methods
+
+There are several denoising methods available.
+
+| Trainer Name         | Module                               | Description                                                                                                                                                                                                   |
+| -------------------- | -------------------------------------| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| MajorityVoteTrainer  |`knodle.trainer.baseline`             | This builds the baseline for all methods. No denoising takes place. The final label will be decided by using a simple majority vote approach and the provided model will be trained with these labels.        |
+| AutoTrainer          |`knodle.trainer`                      | This incorporates all denoising methods currently provided in Knodle. |
+| KnnDenoisingTrainer  |`knodle.trainer.knn_denoising`        | This method looks at the similarities in sentence values. The intuition behind it is that similar samples should be activated by the same rules which is allowed by a smoothness assumption on the target space. Similar sentences will receive the same label matches of the rules. This counteracts the problem of missing rules for certain labels. |
+| WSCrossWeighTrainer  |`knodle.trainer.wscrossweigh`         | This method weighs the training samples basing on how reliable their labels are. The less reliable sentences (i.e. sentences, whose weak labels are possibly wrong) are detected using a DS-CrossWeigh method, which is similar to k-fold cross-validation, and got reduced weights in further training. This counteracts the problem of wrongly classified sentences. |
+| SnorkelTrainer       |`knodle.trainer.snorkel`              | A wrapper of the Snorkel system, which incorporates both generative and discriminative Snorkel steps in a single call.  |
+
+Each of the methods has its own default config file, which will be used in training if no custom config is provided. 
+
+## Details about negative samples
+
+## Tutorials
+
+We also aimed at providing the users with basic tutorials that would explain how to use our framework. All of them are stored in [examples](https://github.com/knodle/knodle/tree/develop/examples/) folder and logically divided into two groups:
+- tutorials that demonstrate how to prepare the input data for Knodle Framework...
+    - ... on the example of a well-known ImdB dataset. A weakly supervised dataset is created by incorporating keywords as weak sources ([link](https://github.com/knodle/knodle/tree/develop/examples/data_preprocessing/imdb_dataset)).
+    - ... on the example of a TAC-based dataset in .conll format. A relation extraction dataset is created using entity pairs from Freebase as weak sources ([link](https://github.com/knodle/knodle/tree/develop/examples/data_preprocessing/tac_based_dataset)).
+- tutorials how to work with Knodle Framework...
+    - ... on the example of AutoTrainer. This trainer is to be called when user wants to train a weak classifier, but has no intention to use any specific denoising method, but rather try all currently provided in Knodle ([link](https://github.com/knodle/knodle/tree/develop/examples/trainer/autotrainer)).
+    - ... on the example of WSCrossWeighTrainer. With this trainer a weak classifier with WSCrossWeigh denoising method will be trained ([link](https://github.com/knodle/knodle/tree/develop/examples/trainer/WSCrossWeigh)).
 
 ## Compatibility
 
-## Licence
+Currently the package is tested on Python 3.7. It is possible to add further versions. The CI/CD pipeline needs to be updated in that case.
+
+## Structure
+
+The structure of the code is as follows
+```
+knodle
+├── knodle
+│    ├── evaluation
+│    ├── model
+│    ├── trainer
+│          ├── baseline
+│          ├── knn_denoising
+│          ├── snorkel
+│          ├── wscrossweigh
+│          └── utils
+│    ├── transformation
+│    └── utils
+├── tests
+│    ├── data
+│    ├── evaluation
+│    ├── trainer
+│          ├── baseline
+│          ├── wscrossweigh
+│          ├── snorkel
+│          └── utils
+│    └── transformation
+└── examples
+     ├── data_preprocessing
+           ├── imdb_dataset
+           └── tac_based_dataset
+     └── training
+           ├── simple_auto_trainer
+           └── wscrossweigh
+
+```
+
+## License
+
+Licensed under the [Apache 2.0 License](LICENSE).
+
+## Contact
+
+If you notices a problem in the code, you can report it by [submitting an issue](<https://github.com/knodle/kndole/issues/new>).
+
+If you want to share your feedback with us or take part in the project, contact us via 
+<knodle@cs.univie.ac.at>. 
+
+And don't forget to follow [@knodle_ai](https://twitter.com/knodle_ai) on Twitter :)
 
 ## Authors
+
+- [Anastasiia Sedova](https://github.com/agsedova)
+- [Andreas Stephan](https://github.com/AndSt)
+- [Marina Speranskaya](https://github.com/marina-sp) 
+- [Alessandro Volpicella](https://github.com/AlessandroVol23)
+- [Benjamin Roth](https://www.benjaminroth.net/)
+
+## Citation
+
+```
+@misc{sedova2021knodle,
+      title={Knodle: Modular Weakly Supervised Learning with PyTorch}, 
+      author={Anastasiia Sedova, Andreas Stephan, Marina Speranskaya, and Benjamin Roth},
+      year={2021},
+      eprint={2104.11557},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+```
+
+## Acknowledgments
+
+This research was funded by the WWTF though the project “Knowledge-infused Deep Learning for Natural Language Processing” (WWTF Vienna Research Group VRG19-008).
diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md
new file mode 100644
index 00000000..3ff25277
--- /dev/null
+++ b/STYLE_GUIDE.md
@@ -0,0 +1,142 @@
+## Development
+
+### Git Flow
+
+We follow the [git flow](https://gist.github.com/digitaljhelms/4287848) approach while developing this application. Basically we have three environments:
+
+1. Develop: All features will be merged into develop and tested from the team.
+2. Staging: After the develop environment was tested and the requested features were implemented a staging / release environment is prepared. On that environment all features should be tested by several end-users.
+3. Main: This is the stage which automatically deploys to the the end-users.
+
+Every new feature will be implemented with a feature branch which follows normally the naming convention of `feature/<ISSUE-NUMBER>`. All feature branches will be merged via a Pull Request after a code review. All default branches are locked for direct pushes.
+
+### Style Guide
+
+We propose to follow all a same style guide within the code to ensure readability. The style guide follows heavily the [Pep 8 Style Guide](https://www.python.org/dev/peps/pep-0008/?). Google has a good example of a its [styleguide](https://google.github.io/styleguide/pyguide.html)
+
+#### 1. Lint
+
+We use `flake8` as a linter.
+
+#### 2. Docstring
+
+- Write docstrings.
+- We need them to automatically create an API documentation.
+- If it is completely obvious what is happening the docstring can be one-line. If the method is also private you can consider not writing docstrings.
+- Use Google's docstring format. You can change it in your IDE (e.g. PyCharm, VsCode) to automatically insert the docstring.
+- Contains:
+  - Short description: What is the function doing
+  - Arguments: All arguments provided
+  - Raises: All exceptions
+- Classes should have a docstring after the class declaration
+
+#### 3. Imports
+
+- Import from most generic (e.g. `os`, `sys`) to least generic (e.g. `knodle.trainer`)
+
+#### 4. Naming
+
+- Please use descriptive names. No 1 letter names and no abbreviations. If mathematical expression is needed for explanation append it to a descriptive name, e.g. `mapping_rules_labels_t`.
+- Private functions start with an underscore:
+
+```python
+def _private_function(...):
+    pass
+```
+
+##### File Names
+
+Guidelines derived from [Guido](https://en.wikipedia.org/wiki/Guido_van_Rossum)'s Recommendations
+
+<table rules="all" border="1" summary="Guidelines from Guido's Recommendations"
+       cellspacing="2" cellpadding="2">
+
+  <tr>
+    <th>Type</th>
+    <th>Public</th>
+    <th>Internal</th>
+  </tr>
+
+  <tr>
+    <td>Packages</td>
+    <td><code>lower_with_under</code></td>
+    <td></td>
+  </tr>
+
+  <tr>
+    <td>Modules</td>
+    <td><code>lower_with_under</code></td>
+    <td><code>_lower_with_under</code></td>
+  </tr>
+
+  <tr>
+    <td>Classes</td>
+    <td><code>CapWords</code></td>
+    <td><code>_CapWords</code></td>
+  </tr>
+
+  <tr>
+    <td>Exceptions</td>
+    <td><code>CapWords</code></td>
+    <td></td>
+  </tr>
+
+  <tr>
+    <td>Functions</td>
+    <td><code>lower_with_under()</code></td>
+    <td><code>_lower_with_under()</code></td>
+  </tr>
+
+  <tr>
+    <td>Global/Class Constants</td>
+    <td><code>CAPS_WITH_UNDER</code></td>
+    <td><code>_CAPS_WITH_UNDER</code></td>
+  </tr>
+
+  <tr>
+    <td>Global/Class Variables</td>
+    <td><code>lower_with_under</code></td>
+    <td><code>_lower_with_under</code></td>
+  </tr>
+
+  <tr>
+    <td>Instance Variables</td>
+    <td><code>lower_with_under</code></td>
+    <td><code>_lower_with_under</code> (protected)</td>
+  </tr>
+
+  <tr>
+    <td>Method Names</td>
+    <td><code>lower_with_under()</code></td>
+    <td><code>_lower_with_under()</code> (protected)</td>
+  </tr>
+
+  <tr>
+    <td>Function/Method Parameters</td>
+    <td><code>lower_with_under</code></td>
+    <td></td>
+  </tr>
+
+  <tr>
+    <td>Local Variables</td>
+    <td><code>lower_with_under</code></td>
+    <td></td>
+  </tr>
+
+</table>
+
+#### 5. Function Length
+
+- Use short and focused functions which should just do one thing
+- Google's recommendation lies at about 40 LOC to think about breaking it apart into more functions
+
+#### 6. Type Annotations
+
+- Use [type annotations](https://www.python.org/dev/peps/pep-0484/) in every function
+
+**BE CONSISTENT**. Review your code before submitting a PR and check if it consists with all other code. It'll be easier in the future for everybody to maintain :slightly_smiling_face:
+
+### Add new requirements
+
+Please add all requirements manually. If it isn't needed to have an explicit version just enter the package name (e.g. `pandas`) without any version.
+
diff --git a/requirementx.txt b/examples/data_preprocessing/__init__.py
similarity index 100%
rename from requirementx.txt
rename to examples/data_preprocessing/__init__.py
diff --git a/examples/data_preprocessing/imdb_dataset/Prepare_weak_IMDB_data.ipynb b/examples/data_preprocessing/imdb_dataset/Prepare_weak_IMDB_data.ipynb
new file mode 100644
index 00000000..0b2da016
--- /dev/null
+++ b/examples/data_preprocessing/imdb_dataset/Prepare_weak_IMDB_data.ipynb
@@ -0,0 +1,1941 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# IMDB Dataset - Create Weak Supervision Sources and Get the Weak Data Annotations\n",
+    "\n",
+    "This notebook shows how to use keywords as a weak supervision source on the example of a well-known IMDB Movie Review dataset, which targets a binary sentiment analysis task.\n",
+    "\n",
+    "The original dataset has gold labels, but we will use these labels only for evaluation purposes, since we want to test models under the weak supervision setting with Knodle. The idea behind it is that you don't have a dataset which is purely labeled with strong supervision (manual) and instead use heuristics (e.g. rules) to obtain a weak labeling. In the following tutorial, we will look for certain keywords expressing positive and negative sentiments that can be helpful to distinguish between classes. Specifically, we use the [Opinion lexicon](https://www.cs.uic.edu/~liub/FBS/sentiment-analysis.html) of the University of Illinois at Chicago.\n",
+    "\n",
+    "First, we load the dataset from a Knodle dataset collection. Then, we will create [Snorkel](https://www.snorkel.org/) labeling functions from two sets of keywords and apply them to the IMDB reviews. Please keep in mind, that keyword matching can be done without Snorkel; however, we enjoy the transparent annotation functionality of this library in our tutorial. \n",
+    "Each labeling function (i.e. keyword) will be further associated with a respective target label. This concludes the annotation step.\n",
+    "\n",
+    "To estimate how good our weak labeling works on its own, we will use the resulting keyword matches together with a basic majority vote model. Finally, the preprocessed data will be saved in a knodle-friendly format, so that other denoising models can be trained with the IMDB dataset."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The IMDB dataset available in the Knodle collection was downloaded from [Kaggle](https://www.kaggle.com/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews) in January 2021. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Imports\n",
+    "\n",
+    "Lets make some basic imports "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:32.383150Z",
+     "start_time": "2021-04-13T14:47:28.850762Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from tqdm import tqdm\n",
+    "from typing import List\n",
+    "\n",
+    "import pandas as pd \n",
+    "import numpy as np \n",
+    "\n",
+    "from bs4 import BeautifulSoup\n",
+    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS, CountVectorizer\n",
+    "from snorkel.labeling import LabelingFunction, PandasLFApplier, filter_unlabeled_dataframe, LFAnalysis\n",
+    "from snorkel.labeling.model import MajorityLabelVoter, LabelModel\n",
+    "\n",
+    "# client to access the dataset collection\n",
+    "from minio import Minio\n",
+    "client = Minio(\"knodle.dm.univie.ac.at\", secure=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:32.393501Z",
+     "start_time": "2021-04-13T14:47:32.386143Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/mounts/Users/student/speranskaya/.local/lib/python3.8/site-packages/tqdm/std.py:699: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
+      "  from pandas import Panel\n",
+      "<ipython-input-3-b8e999b6de0d>:3: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
+      "  pd.set_option('display.max_colwidth', -1)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Init\n",
+    "tqdm.pandas()\n",
+    "pd.set_option('display.max_colwidth', -1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:32.398374Z",
+     "start_time": "2021-04-13T14:47:32.395786Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Constants for Snorkel labeling functions\n",
+    "POSITIVE = 1\n",
+    "NEGATIVE = 0\n",
+    "ABSTAIN = -1\n",
+    "COLUMN_WITH_TEXT = \"reviews_preprocessed\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Download the dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:32.403548Z",
+     "start_time": "2021-04-13T14:47:32.400510Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# define the path to the folder where the data will be stored\n",
+    "data_path = \"../../../data_from_minio/imdb\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Together with the IMDB data, let us also collect the keywords that we will need later."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:36.498235Z",
+     "start_time": "2021-04-13T14:47:32.405549Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 3/3 [00:04<00:00,  1.36s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "files = [(\"IMDB Dataset.csv\",), (\"keywords\", \"negative-words.txt\"), (\"keywords\", \"positive-words.txt\")]\n",
+    "\n",
+    "for file in tqdm(files):\n",
+    "    client.fget_object(\n",
+    "        bucket_name=\"knodle\",\n",
+    "        object_name=os.path.join(\"datasets/imdb\", *file),\n",
+    "        file_path=os.path.join(data_path, file[-1]),\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Preview dataset\n",
+    "\n",
+    "After downloading and unpacking the dataset we can have a first look at it and work with it."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:37.413783Z",
+     "start_time": "2021-04-13T14:47:36.500749Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "imdb_dataset_raw = pd.read_csv(os.path.join(data_path, \"IMDB Dataset.csv\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:37.443703Z",
+     "start_time": "2021-04-13T14:47:37.416825Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>review</th>\n",
+       "      <th>sentiment</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>One of the other reviewers has mentioned that after watching just 1 Oz episode you'll be hooked. They are right, as this is exactly what happened with me.&lt;br /&gt;&lt;br /&gt;The first thing that struck me about Oz was its brutality and unflinching scenes of violence, which set in right from the word GO. Trust me, this is not a show for the faint hearted or timid. This show pulls no punches with regards to drugs, sex or violence. Its is hardcore, in the classic use of the word.&lt;br /&gt;&lt;br /&gt;It is called OZ as that is the nickname given to the Oswald Maximum Security State Penitentary. It focuses mainly on Emerald City, an experimental section of the prison where all the cells have glass fronts and face inwards, so privacy is not high on the agenda. Em City is home to many..Aryans, Muslims, gangstas, Latinos, Christians, Italians, Irish and more....so scuffles, death stares, dodgy dealings and shady agreements are never far away.&lt;br /&gt;&lt;br /&gt;I would say the main appeal of the show is due to the fact that it goes where other shows wouldn't dare. Forget pretty pictures painted for mainstream audiences, forget charm, forget romance...OZ doesn't mess around. The first episode I ever saw struck me as so nasty it was surreal, I couldn't say I was ready for it, but as I watched more, I developed a taste for Oz, and got accustomed to the high levels of graphic violence. Not just violence, but injustice (crooked guards who'll be sold out for a nickel, inmates who'll kill on order and get away with it, well mannered, middle class inmates being turned into prison bitches due to their lack of street skills or prison experience) Watching Oz, you may become comfortable with what is uncomfortable viewing....thats if you can get in touch with your darker side.</td>\n",
+       "      <td>positive</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              review  \\\n",
+       "0  One of the other reviewers has mentioned that after watching just 1 Oz episode you'll be hooked. They are right, as this is exactly what happened with me.<br /><br />The first thing that struck me about Oz was its brutality and unflinching scenes of violence, which set in right from the word GO. Trust me, this is not a show for the faint hearted or timid. This show pulls no punches with regards to drugs, sex or violence. Its is hardcore, in the classic use of the word.<br /><br />It is called OZ as that is the nickname given to the Oswald Maximum Security State Penitentary. It focuses mainly on Emerald City, an experimental section of the prison where all the cells have glass fronts and face inwards, so privacy is not high on the agenda. Em City is home to many..Aryans, Muslims, gangstas, Latinos, Christians, Italians, Irish and more....so scuffles, death stares, dodgy dealings and shady agreements are never far away.<br /><br />I would say the main appeal of the show is due to the fact that it goes where other shows wouldn't dare. Forget pretty pictures painted for mainstream audiences, forget charm, forget romance...OZ doesn't mess around. The first episode I ever saw struck me as so nasty it was surreal, I couldn't say I was ready for it, but as I watched more, I developed a taste for Oz, and got accustomed to the high levels of graphic violence. Not just violence, but injustice (crooked guards who'll be sold out for a nickel, inmates who'll kill on order and get away with it, well mannered, middle class inmates being turned into prison bitches due to their lack of street skills or prison experience) Watching Oz, you may become comfortable with what is uncomfortable viewing....thats if you can get in touch with your darker side.   \n",
+       "\n",
+       "  sentiment  \n",
+       "0  positive  "
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "imdb_dataset_raw.head(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:37.461163Z",
+     "start_time": "2021-04-13T14:47:37.445292Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>review</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>sentiment</th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>negative</th>\n",
+       "      <td>25000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>positive</th>\n",
+       "      <td>25000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           review\n",
+       "sentiment        \n",
+       "negative   25000 \n",
+       "positive   25000 "
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "imdb_dataset_raw.groupby('sentiment').count()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:37.471947Z",
+     "start_time": "2021-04-13T14:47:37.463498Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "review       0\n",
+       "sentiment    0\n",
+       "dtype: int64"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "imdb_dataset_raw.isna().sum()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Preprocess dataset\n",
+    "\n",
+    "Now lets take some basic preprocessing steps"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Remove Stopwords\n",
+    "\n",
+    "It could be a reasonable step for some classifiers, but since we use BERT among other approaches, we want to keep the sentence structure and hence do not remove stopwords just yet."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Remove HTML Tags\n",
+    "\n",
+    "The dataset contains many HTML tags. We'll remove them"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.060677Z",
+     "start_time": "2021-04-13T14:47:37.473837Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def strip_html(text):\n",
+    "    soup = BeautifulSoup(text, \"html.parser\")\n",
+    "    return soup.get_text()\n",
+    "\n",
+    "imdb_dataset_raw[COLUMN_WITH_TEXT] = imdb_dataset_raw[\"review\"].apply(\n",
+    "    lambda x: strip_html(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.067633Z",
+     "start_time": "2021-04-13T14:47:44.062579Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0    One of the other reviewers has mentioned that after watching just 1 Oz episode you'll be hooked. They are right, as this is exactly what happened with me.The first thing that struck me about Oz was its brutality and unflinching scenes of violence, which set in right from the word GO. Trust me, this is not a show for the faint hearted or timid. This show pulls no punches with regards to drugs, sex or violence. Its is hardcore, in the classic use of the word.It is called OZ as that is the nickname given to the Oswald Maximum Security State Penitentary. It focuses mainly on Emerald City, an experimental section of the prison where all the cells have glass fronts and face inwards, so privacy is not high on the agenda. Em City is home to many..Aryans, Muslims, gangstas, Latinos, Christians, Italians, Irish and more....so scuffles, death stares, dodgy dealings and shady agreements are never far away.I would say the main appeal of the show is due to the fact that it goes where other shows wouldn't dare. Forget pretty pictures painted for mainstream audiences, forget charm, forget romance...OZ doesn't mess around. The first episode I ever saw struck me as so nasty it was surreal, I couldn't say I was ready for it, but as I watched more, I developed a taste for Oz, and got accustomed to the high levels of graphic violence. Not just violence, but injustice (crooked guards who'll be sold out for a nickel, inmates who'll kill on order and get away with it, well mannered, middle class inmates being turned into prison bitches due to their lack of street skills or prison experience) Watching Oz, you may become comfortable with what is uncomfortable viewing....thats if you can get in touch with your darker side.\n",
+       "Name: reviews_preprocessed, dtype: object"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "imdb_dataset_raw[COLUMN_WITH_TEXT].head(1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Keywords\n",
+    "\n",
+    "For weak supervision sources we use sentiment keyword lists for positive and negative words.\n",
+    "\n",
+    "We have downloaded them from the Knodle collection earlier, with the IMDB dataset. \n",
+    "\n",
+    "After parsing the keywords from separate files, they are stored in a pd.DataFrame with the corresponding sentiment as \"label\"."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.083452Z",
+     "start_time": "2021-04-13T14:47:44.069134Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>keyword</th>\n",
+       "      <th>label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>a+</td>\n",
+       "      <td>positive</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>abound</td>\n",
+       "      <td>positive</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  keyword     label\n",
+       "0  a+      positive\n",
+       "1  abound  positive"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "positive_keywords = pd.read_csv(os.path.join(data_path, \"positive-words.txt\"), sep=\" \", header=None, error_bad_lines=False, skiprows=30)\n",
+    "positive_keywords.columns = [\"keyword\"]\n",
+    "positive_keywords[\"label\"] = \"positive\"\n",
+    "positive_keywords.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.095630Z",
+     "start_time": "2021-04-13T14:47:44.084754Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>keyword</th>\n",
+       "      <th>label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2-faced</td>\n",
+       "      <td>negative</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2-faces</td>\n",
+       "      <td>negative</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   keyword     label\n",
+       "0  2-faced  negative\n",
+       "1  2-faces  negative"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "negative_keywords = pd.read_csv(os.path.join(data_path, \"negative-words.txt\"),\n",
+    "                                sep=\" \", header=None, error_bad_lines=False,  encoding='latin-1', skiprows=30)\n",
+    "negative_keywords.columns = [\"keyword\"]\n",
+    "negative_keywords[\"label\"] = \"negative\"\n",
+    "negative_keywords.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.103255Z",
+     "start_time": "2021-04-13T14:47:44.096977Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "negative    4783\n",
+       "positive    2006\n",
+       "Name: label, dtype: int64"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_keywords = pd.concat([positive_keywords, negative_keywords])\n",
+    "all_keywords.label.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.111447Z",
+     "start_time": "2021-04-13T14:47:44.104640Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "negative    4780\n",
+       "positive    2006\n",
+       "Name: label, dtype: int64"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# remove overlap of keywords between two sentiments\n",
+    "all_keywords.drop_duplicates('keyword',inplace=True)\n",
+    "all_keywords.label.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Labeling Functions\n",
+    "\n",
+    "Now we start to build labeling functions with Snorkel with these keywords and check the coverage.\n",
+    "\n",
+    "This is an iterative process of course so we surely have to add more keywords and regulary expressions ;-) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.114774Z",
+     "start_time": "2021-04-13T14:47:44.112751Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def keyword_lookup(x, keyword, label):\n",
+    "    return label if keyword in x[COLUMN_WITH_TEXT].lower() else ABSTAIN\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.120497Z",
+     "start_time": "2021-04-13T14:47:44.115958Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def make_keyword_lf(keyword: str, label: str) -> LabelingFunction:\n",
+    "    \"\"\"\n",
+    "    Creates labeling function based on keyword.\n",
+    "    Args:\n",
+    "        keyword: what keyword should be look for\n",
+    "        label: what label does this keyword imply\n",
+    "\n",
+    "    Returns: LabelingFunction object\n",
+    "\n",
+    "    \"\"\"\n",
+    "    return LabelingFunction(\n",
+    "        name=f\"keyword_{keyword}\",\n",
+    "        f=keyword_lookup,\n",
+    "        resources=dict(keyword=keyword, label=label),\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.127252Z",
+     "start_time": "2021-04-13T14:47:44.122211Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def create_labeling_functions(keywords: pd.DataFrame) -> np.ndarray:\n",
+    "    \"\"\"\n",
+    "    Create Labeling Functions based on the columns keyword and regex. Appends column lf to df.\n",
+    "\n",
+    "    Args:\n",
+    "        keywords: DataFrame with processed keywords\n",
+    "\n",
+    "    Returns:\n",
+    "        All labeling functions. 1d Array with shape: (number_of_lfs x 1)\n",
+    "    \"\"\"\n",
+    "    keywords = keywords.assign(lf=keywords.progress_apply(\n",
+    "        lambda x:make_keyword_lf(x.keyword, x.label_id), axis=1\n",
+    "    ))\n",
+    "    lfs = keywords.lf.values\n",
+    "    return lfs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.133856Z",
+     "start_time": "2021-04-13T14:47:44.129057Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "all_keywords[\"label_id\"] = all_keywords.label.map({'positive':POSITIVE, 'negative':NEGATIVE})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.144096Z",
+     "start_time": "2021-04-13T14:47:44.135120Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>keyword</th>\n",
+       "      <th>label</th>\n",
+       "      <th>label_id</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>a+</td>\n",
+       "      <td>positive</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>abound</td>\n",
+       "      <td>positive</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>abounds</td>\n",
+       "      <td>positive</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>abundance</td>\n",
+       "      <td>positive</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>abundant</td>\n",
+       "      <td>positive</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4778</th>\n",
+       "      <td>zaps</td>\n",
+       "      <td>negative</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4779</th>\n",
+       "      <td>zealot</td>\n",
+       "      <td>negative</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4780</th>\n",
+       "      <td>zealous</td>\n",
+       "      <td>negative</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4781</th>\n",
+       "      <td>zealously</td>\n",
+       "      <td>negative</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4782</th>\n",
+       "      <td>zombie</td>\n",
+       "      <td>negative</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>6786 rows × 3 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        keyword     label  label_id\n",
+       "0     a+         positive  1       \n",
+       "1     abound     positive  1       \n",
+       "2     abounds    positive  1       \n",
+       "3     abundance  positive  1       \n",
+       "4     abundant   positive  1       \n",
+       "...        ...        ... ..       \n",
+       "4778  zaps       negative  0       \n",
+       "4779  zealot     negative  0       \n",
+       "4780  zealous    negative  0       \n",
+       "4781  zealously  negative  0       \n",
+       "4782  zombie     negative  0       \n",
+       "\n",
+       "[6786 rows x 3 columns]"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_keywords"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.331390Z",
+     "start_time": "2021-04-13T14:47:44.145328Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 6786/6786 [00:00<00:00, 37675.23it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "labeling_functions = create_labeling_functions(all_keywords)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T14:47:44.335612Z",
+     "start_time": "2021-04-13T14:47:44.332724Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([LabelingFunction keyword_a+, Preprocessors: [],\n",
+       "       LabelingFunction keyword_abound, Preprocessors: [],\n",
+       "       LabelingFunction keyword_abounds, Preprocessors: [], ...,\n",
+       "       LabelingFunction keyword_zealous, Preprocessors: [],\n",
+       "       LabelingFunction keyword_zealously, Preprocessors: [],\n",
+       "       LabelingFunction keyword_zombie, Preprocessors: []], dtype=object)"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "labeling_functions"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Apply Labeling Functions\n",
+    "\n",
+    "Now lets apply all labeling functions on our reviews and check some statistics."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:16:26.457239Z",
+     "start_time": "2021-04-13T14:47:44.336718Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/mounts/Users/student/speranskaya/.local/lib/python3.8/site-packages/tqdm/std.py:699: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
+      "  from pandas import Panel\n",
+      "100%|██████████| 50000/50000 [28:37<00:00, 29.11it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "applier = PandasLFApplier(lfs=labeling_functions)\n",
+    "applied_lfs = applier.apply(df=imdb_dataset_raw)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:16:26.462239Z",
+     "start_time": "2021-04-13T15:16:26.458978Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[-1, -1, -1, ..., -1, -1, -1],\n",
+       "       [-1, -1, -1, ..., -1, -1, -1],\n",
+       "       [-1, -1, -1, ..., -1, -1, -1],\n",
+       "       ...,\n",
+       "       [-1, -1, -1, ..., -1, -1, -1],\n",
+       "       [-1, -1, -1, ..., -1, -1, -1],\n",
+       "       [-1, -1, -1, ..., -1, -1, -1]])"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "applied_lfs"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we have a matrix with all labeling functions applied. This matrix has the shape $(instances \\times labeling functions)$"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:16:26.469326Z",
+     "start_time": "2021-04-13T15:16:26.463482Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Shape of applied labeling functions:  (50000, 6786)\n",
+      "Number of reviews 50000\n",
+      "Number of labeling functions 6786\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Shape of applied labeling functions: \", applied_lfs.shape)\n",
+    "print(\"Number of reviews\", len(imdb_dataset_raw))\n",
+    "print(\"Number of labeling functions\", len(labeling_functions))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Analysis\n",
+    "\n",
+    "Now we can analyse some basic stats about our labeling functions. The main figures are:\n",
+    "\n",
+    "- Coverage: How many labeling functions match at all\n",
+    "- Overlaps: How many labeling functions overlap with each other (e.g. awesome and amazing)\n",
+    "- Conflicts: How many labeling functions overlap and have different labels (e.g. awesome and bad)\n",
+    "- Correct: Correct LFs\n",
+    "- Incorrect: Incorrect Lfs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:17:03.804541Z",
+     "start_time": "2021-04-13T15:16:26.471086Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>j</th>\n",
+       "      <th>Polarity</th>\n",
+       "      <th>Coverage</th>\n",
+       "      <th>Overlaps</th>\n",
+       "      <th>Conflicts</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>keyword_a+</th>\n",
+       "      <td>0</td>\n",
+       "      <td>[1]</td>\n",
+       "      <td>0.00102</td>\n",
+       "      <td>0.00102</td>\n",
+       "      <td>0.00102</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_abound</th>\n",
+       "      <td>1</td>\n",
+       "      <td>[1]</td>\n",
+       "      <td>0.00264</td>\n",
+       "      <td>0.00264</td>\n",
+       "      <td>0.00264</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_abounds</th>\n",
+       "      <td>2</td>\n",
+       "      <td>[1]</td>\n",
+       "      <td>0.00052</td>\n",
+       "      <td>0.00052</td>\n",
+       "      <td>0.00052</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_abundance</th>\n",
+       "      <td>3</td>\n",
+       "      <td>[1]</td>\n",
+       "      <td>0.00182</td>\n",
+       "      <td>0.00182</td>\n",
+       "      <td>0.00182</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_abundant</th>\n",
+       "      <td>4</td>\n",
+       "      <td>[1]</td>\n",
+       "      <td>0.00114</td>\n",
+       "      <td>0.00114</td>\n",
+       "      <td>0.00114</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_zaps</th>\n",
+       "      <td>6781</td>\n",
+       "      <td>[0]</td>\n",
+       "      <td>0.00012</td>\n",
+       "      <td>0.00012</td>\n",
+       "      <td>0.00012</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_zealot</th>\n",
+       "      <td>6782</td>\n",
+       "      <td>[0]</td>\n",
+       "      <td>0.00048</td>\n",
+       "      <td>0.00048</td>\n",
+       "      <td>0.00048</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_zealous</th>\n",
+       "      <td>6783</td>\n",
+       "      <td>[0]</td>\n",
+       "      <td>0.00092</td>\n",
+       "      <td>0.00092</td>\n",
+       "      <td>0.00092</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_zealously</th>\n",
+       "      <td>6784</td>\n",
+       "      <td>[0]</td>\n",
+       "      <td>0.00006</td>\n",
+       "      <td>0.00006</td>\n",
+       "      <td>0.00006</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>keyword_zombie</th>\n",
+       "      <td>6785</td>\n",
+       "      <td>[0]</td>\n",
+       "      <td>0.01508</td>\n",
+       "      <td>0.01508</td>\n",
+       "      <td>0.01508</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>6786 rows × 5 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                      j Polarity  Coverage  Overlaps  Conflicts\n",
+       "keyword_a+         0     [1]      0.00102   0.00102   0.00102  \n",
+       "keyword_abound     1     [1]      0.00264   0.00264   0.00264  \n",
+       "keyword_abounds    2     [1]      0.00052   0.00052   0.00052  \n",
+       "keyword_abundance  3     [1]      0.00182   0.00182   0.00182  \n",
+       "keyword_abundant   4     [1]      0.00114   0.00114   0.00114  \n",
+       "...               ..     ...          ...       ...       ...  \n",
+       "keyword_zaps       6781  [0]      0.00012   0.00012   0.00012  \n",
+       "keyword_zealot     6782  [0]      0.00048   0.00048   0.00048  \n",
+       "keyword_zealous    6783  [0]      0.00092   0.00092   0.00092  \n",
+       "keyword_zealously  6784  [0]      0.00006   0.00006   0.00006  \n",
+       "keyword_zombie     6785  [0]      0.01508   0.01508   0.01508  \n",
+       "\n",
+       "[6786 rows x 5 columns]"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "LFAnalysis(L=applied_lfs, lfs=labeling_functions).lf_summary()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:17:41.191508Z",
+     "start_time": "2021-04-13T15:17:03.807827Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "lf_analysis = LFAnalysis(L=applied_lfs, lfs=labeling_functions).lf_summary()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:17:41.277784Z",
+     "start_time": "2021-04-13T15:17:41.193281Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>j</th>\n",
+       "      <td>3392.500000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Coverage</th>\n",
+       "      <td>0.005163</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Overlaps</th>\n",
+       "      <td>0.005163</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Conflicts</th>\n",
+       "      <td>0.005159</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                     0\n",
+       "j          3392.500000\n",
+       "Coverage   0.005163   \n",
+       "Overlaps   0.005163   \n",
+       "Conflicts  0.005159   "
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.DataFrame(lf_analysis.mean())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:17:41.288092Z",
+     "start_time": "2021-04-13T15:17:41.279033Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>0</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>j</th>\n",
+       "      <td>3392.50000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Coverage</th>\n",
+       "      <td>0.00052</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Overlaps</th>\n",
+       "      <td>0.00052</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Conflicts</th>\n",
+       "      <td>0.00052</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                    0\n",
+       "j          3392.50000\n",
+       "Coverage   0.00052   \n",
+       "Overlaps   0.00052   \n",
+       "Conflicts  0.00052   "
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.DataFrame(lf_analysis.median())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Lets have a look at some examples that were labeled by a positive keyword. You can see, that the true label for some of them is negative."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:17:41.293277Z",
+     "start_time": "2021-04-13T15:17:41.289552Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "keyword     loyalty \n",
+       "label       positive\n",
+       "label_id    1       \n",
+       "Name: 1110, dtype: object"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# consider 50th keyword\n",
+    "all_keywords.iloc[1110]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T15:17:41.324488Z",
+     "start_time": "2021-04-13T15:17:41.294657Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>reviews_preprocessed</th>\n",
+       "      <th>sentiment</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>31052</th>\n",
+       "      <td>I finally got my wish to see this one in a cinema. I'd seen Fritz Lang's film on video some years ago. I'd been hoping that ideal screening conditions would work their magic.Conditions were ideal at Cinematheque Ontario. Pristine full-length print. Intertitles in the original Gothic-script German with simultaneous English translation, accurate without being too literal. Live piano accompaniment. Ideal.The film's magic sputtered for a little while but ultimately failed to catch, at least for me.This film bears no real relation to Wagner's Ring cycle as I already knew but some may not. Wagner had adapted the 13th c. Niebelungenlied to his own purposes. Part I of Fritz Lang's epic -- \"Siegfried\" -- has much that will be familiar to listeners of Wagner however.\"Kriemhild's Revenge\" is the story of Siegfried's wife Kriemhild, her marriage to King Etzel (Attila) the Hun, and her desire for revenge against Hagen and Gunther, the rechristened Nibelungs, for the murder of Siegfried. The spectacular conflagration in this film presumably evolved and expanded in the Wagnerian mythos into his Götterdämmerung, his Twilight of the Gods, and the end of Valhalla. This film remains earthbound.Most of the film is spectacular. The massive sets rival those of \"Cabiria\" (1914), which inspired Griffith's \"Intolerance\" (1916). Their decoration sets a new benchmark in barbaric splendour. There's a huge cast of scarred, mangy Huns and Art Deco Burgundians. And battles. Battles that never seem to end in fact.Kriemhild is very successful in her plan of revenge. She manages to destroy all around her. Her loyalty to her martyred Siegfried seems not to stem so much from love, or devotion, but from something closer to psychosis. Lady Macbeth cried out, \"Unsex me here.\" She knew she was emotionally unprepared for what she needed to do. But Kriemhild displays no normal human emotions, and certainly nothing one equates with the feminine principle. She is already \"top full of direst cruelty\", to borrow Shakespeare's phrase, from the outset. Margarethe Schön and her director convey this with a glower. I don't want to exaggerate, but that glower is virtually the only expression ever to \"animate\" Kriemhild's face. It's the ultimate in one-note performances. It's clearly intentional however, not simply a case of poor acting.What we have then on offer is a one-dimensional sketch of an avenging Fury. Some might see Kriemhild as an empowered heroine. I just see the film as misogynistic.</td>\n",
+       "      <td>positive</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1798</th>\n",
+       "      <td>\"Laugh, Clown Laugh\" released in 1928, stars the legendary Lon Chaney as a circus clown named Tito. Tito has raised a foundling (a young and beautiful Loretta Young) to adulthood and names her Simonetta. Tito has raised the girl in the circus life, and she has become an accomplished ballerina. While Chaney gives his usual great performance, I could not get past the fact that Tito, now well into middle age, has the hots for the young Simonetta. Although he is not her biological father, he has raised her like a daughter. That kind of \"ick\" factor permeates throughout the film. Tito competes for Simonetta's affections with a young and handsome 'Count' Luigi (Nils Asther). Simonetta clearly falls for the young man, but feels guilt about abandoning Tito (out of loyalty, not romantic love). The whole premise of the film is ridiculous, and I find it amazing that no one in the film tells Tito what a stupid old fool he is being (until he reveals it himself at the end). The film is noteworthy only because of Loretta Young, who would go on to have a great career. While I adore Chaney's brilliance as an actor, this whole film seems off to me and just downright creepy.</td>\n",
+       "      <td>negative</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           reviews_preprocessed  \\\n",
+       "31052  I finally got my wish to see this one in a cinema. I'd seen Fritz Lang's film on video some years ago. I'd been hoping that ideal screening conditions would work their magic.Conditions were ideal at Cinematheque Ontario. Pristine full-length print. Intertitles in the original Gothic-script German with simultaneous English translation, accurate without being too literal. Live piano accompaniment. Ideal.The film's magic sputtered for a little while but ultimately failed to catch, at least for me.This film bears no real relation to Wagner's Ring cycle as I already knew but some may not. Wagner had adapted the 13th c. Niebelungenlied to his own purposes. Part I of Fritz Lang's epic -- \"Siegfried\" -- has much that will be familiar to listeners of Wagner however.\"Kriemhild's Revenge\" is the story of Siegfried's wife Kriemhild, her marriage to King Etzel (Attila) the Hun, and her desire for revenge against Hagen and Gunther, the rechristened Nibelungs, for the murder of Siegfried. The spectacular conflagration in this film presumably evolved and expanded in the Wagnerian mythos into his Götterdämmerung, his Twilight of the Gods, and the end of Valhalla. This film remains earthbound.Most of the film is spectacular. The massive sets rival those of \"Cabiria\" (1914), which inspired Griffith's \"Intolerance\" (1916). Their decoration sets a new benchmark in barbaric splendour. There's a huge cast of scarred, mangy Huns and Art Deco Burgundians. And battles. Battles that never seem to end in fact.Kriemhild is very successful in her plan of revenge. She manages to destroy all around her. Her loyalty to her martyred Siegfried seems not to stem so much from love, or devotion, but from something closer to psychosis. Lady Macbeth cried out, \"Unsex me here.\" She knew she was emotionally unprepared for what she needed to do. But Kriemhild displays no normal human emotions, and certainly nothing one equates with the feminine principle. She is already \"top full of direst cruelty\", to borrow Shakespeare's phrase, from the outset. Margarethe Schön and her director convey this with a glower. I don't want to exaggerate, but that glower is virtually the only expression ever to \"animate\" Kriemhild's face. It's the ultimate in one-note performances. It's clearly intentional however, not simply a case of poor acting.What we have then on offer is a one-dimensional sketch of an avenging Fury. Some might see Kriemhild as an empowered heroine. I just see the film as misogynistic.   \n",
+       "1798   \"Laugh, Clown Laugh\" released in 1928, stars the legendary Lon Chaney as a circus clown named Tito. Tito has raised a foundling (a young and beautiful Loretta Young) to adulthood and names her Simonetta. Tito has raised the girl in the circus life, and she has become an accomplished ballerina. While Chaney gives his usual great performance, I could not get past the fact that Tito, now well into middle age, has the hots for the young Simonetta. Although he is not her biological father, he has raised her like a daughter. That kind of \"ick\" factor permeates throughout the film. Tito competes for Simonetta's affections with a young and handsome 'Count' Luigi (Nils Asther). Simonetta clearly falls for the young man, but feels guilt about abandoning Tito (out of loyalty, not romantic love). The whole premise of the film is ridiculous, and I find it amazing that no one in the film tells Tito what a stupid old fool he is being (until he reveals it himself at the end). The film is noteworthy only because of Loretta Young, who would go on to have a great career. While I adore Chaney's brilliance as an actor, this whole film seems off to me and just downright creepy.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
+       "\n",
+       "      sentiment  \n",
+       "31052  positive  \n",
+       "1798   negative  "
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# sample 2 random examples where the 50th LF assigned a positive label\n",
+    "imdb_dataset_raw.iloc[applied_lfs[:, 1110] == POSITIVE, :].sample(2, random_state=1).loc[:, [COLUMN_WITH_TEXT,'sentiment']]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Transform rule matches\n",
+    "\n",
+    "To work with knodle the dataset needs to be transformed into a binary array $Z$\n",
+    "\n",
+    "(shape: `#instances x # rules`).\n",
+    "\n",
+    "Where a cell $Z_{ij}$ means that for instance i\n",
+    "\n",
+    "    0 -> Rule j didn't match \n",
+    "    1 -> Rule j matched\n",
+    "\n",
+    "Furthermore, we need a matrix `mapping_rule_labels_t` which has a mapping of all rules to labels, stored in a binary manner as well \n",
+    "\n",
+    "(shape `#rules x #labels`)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:15:05.480853Z",
+     "start_time": "2021-04-13T18:14:43.147738Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from knodle.transformation.rule_label_format import transform_snorkel_matrix_to_z_t\n",
+    "\n",
+    "rule_matches_z, mapping_rules_labels_t = transform_snorkel_matrix_to_z_t(applied_lfs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Majority vote\n",
+    "\n",
+    "Now we make a majority vote based on all rule matches. First we get the `rule_counts` by multiplying `rule_matches_z` with the `mapping_rules_labels_t`, then we divide it sumwise by the sum to get a probability value. In the end we counteract the divide with zero issue by setting all nan values to zero. All this happens in the `z_t_matrices_to_majority_vote_labels` function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:15:13.892018Z",
+     "start_time": "2021-04-13T18:15:13.863351Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from knodle.transformation.majority import z_t_matrices_to_majority_vote_labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:15:17.263682Z",
+     "start_time": "2021-04-13T18:15:14.831722Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/mounts/work/speranskaya/projects/knodle/knodle/knodle/transformation/majority.py:74: RuntimeWarning: invalid value encountered in true_divide\n",
+      "  rule_counts_probs = rule_counts / rule_counts.sum(axis=1).reshape(-1, 1)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# the ties are resolved randomly internally, so the predictions might slightly vary\n",
+    "pred_labels = z_t_matrices_to_majority_vote_labels(rule_matches_z, mapping_rules_labels_t)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:19:19.421532Z",
+     "start_time": "2021-04-13T18:19:19.407778Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Accuracy of majority vote on weak labeling: 0.66208'"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# accuracy of the weak labels\n",
+    "acc = (pred_labels == imdb_dataset_raw.sentiment.map({'positive':POSITIVE, 'negative':NEGATIVE})).mean()\n",
+    "f\"Accuracy of majority vote on weak labeling: {acc}\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Make splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:19:27.417565Z",
+     "start_time": "2021-04-13T18:19:27.387409Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from scipy.sparse import csr_matrix\n",
+    "from sklearn.model_selection import train_test_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:19:30.439040Z",
+     "start_time": "2021-04-13T18:19:28.828646Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# we want to save the matrix a sparse format, so we convert once before it gets split\n",
+    "rule_matches_sparse = csr_matrix(rule_matches_z)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:19:38.621687Z",
+     "start_time": "2021-04-13T18:19:38.587135Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# adjust DataFrame format to Knodle standard\n",
+    "imdb_dataset_formatted = pd.DataFrame({\n",
+    "    \"sample\": imdb_dataset_raw[COLUMN_WITH_TEXT].values,\n",
+    "    \"label\": imdb_dataset_raw.sentiment.map({'positive':POSITIVE, 'negative':NEGATIVE}).values\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:19:55.950465Z",
+     "start_time": "2021-04-13T18:19:55.898728Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# make splits for samples and weak annotation matrix\n",
+    "rest_df, test_df, rest_rule_matches_sparse_z, test_rule_matches_sparse_z = train_test_split(imdb_dataset_formatted, rule_matches_sparse, test_size=5000, random_state=42)\n",
+    "train_df, dev_df, train_rule_matches_sparse_z, dev_rule_matches_sparse_z = train_test_split(rest_df, rest_rule_matches_sparse_z, test_size=5000, random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:20:05.283684Z",
+     "start_time": "2021-04-13T18:20:05.274375Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/mounts/Users/student/speranskaya/.local/lib/python3.8/site-packages/pandas/core/frame.py:4162: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  return super().drop(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# drop labels for the train split\n",
+    "train_df.drop(columns=[\"label\"], inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:20:37.281249Z",
+     "start_time": "2021-04-13T18:20:37.270576Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sample</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>23844</th>\n",
+       "      <td>I have no idea what these people were thinking when they made this film. No plot, very limited action, and what is with the 3rd person commentary throughout the film???? Instead of running around the planet to shoot on all of these locations, they should have spent some money on script writing and actors. What acting there was, was lousy. This was 90 minutes of my life I will never be able to get back. I should bill the director for the cost of renting this film. To the director and the writers of this film....please quit now. This film should have a tag on the front of it saying beware of boredom. The only good thing I can say about this film, is the computer generation. It's OK as generation is. This movie should never have a sequel....ever.</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  sample\n",
+       "23844  I have no idea what these people were thinking when they made this film. No plot, very limited action, and what is with the 3rd person commentary throughout the film???? Instead of running around the planet to shoot on all of these locations, they should have spent some money on script writing and actors. What acting there was, was lousy. This was 90 minutes of my life I will never be able to get back. I should bill the director for the cost of renting this film. To the director and the writers of this film....please quit now. This film should have a tag on the front of it saying beware of boredom. The only good thing I can say about this film, is the computer generation. It's OK as generation is. This movie should never have a sequel....ever."
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.head(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:20:42.719580Z",
+     "start_time": "2021-04-13T18:20:42.709427Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sample</th>\n",
+       "      <th>label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>33553</th>\n",
+       "      <td>I really liked this Summerslam due to the look of the arena, the curtains and just the look overall was interesting to me for some reason. Anyways, this could have been one of the best Summerslam's ever if the WWF didn't have Lex Luger in the main event against Yokozuna, now for it's time it was ok to have a huge fat man vs a strong man but I'm glad times have changed. It was a terrible main event just like every match Luger is in is terrible. Other matches on the card were Razor Ramon vs Ted Dibiase, Steiner Brothers vs Heavenly Bodies, Shawn Michaels vs Curt Hening, this was the event where Shawn named his big monster of a body guard Diesel, IRS vs 1-2-3 Kid, Bret Hart first takes on Doink then takes on Jerry Lawler and stuff with the Harts and Lawler was always very interesting, then Ludvig Borga destroyed Marty Jannetty, Undertaker took on Giant Gonzalez in another terrible match, The Smoking Gunns and Tatanka took on Bam Bam Bigelow and the Headshrinkers, and Yokozuna defended the world title against Lex Luger this match was boring and it has a terrible ending. However it deserves 8/10</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    sample  \\\n",
+       "33553  I really liked this Summerslam due to the look of the arena, the curtains and just the look overall was interesting to me for some reason. Anyways, this could have been one of the best Summerslam's ever if the WWF didn't have Lex Luger in the main event against Yokozuna, now for it's time it was ok to have a huge fat man vs a strong man but I'm glad times have changed. It was a terrible main event just like every match Luger is in is terrible. Other matches on the card were Razor Ramon vs Ted Dibiase, Steiner Brothers vs Heavenly Bodies, Shawn Michaels vs Curt Hening, this was the event where Shawn named his big monster of a body guard Diesel, IRS vs 1-2-3 Kid, Bret Hart first takes on Doink then takes on Jerry Lawler and stuff with the Harts and Lawler was always very interesting, then Ludvig Borga destroyed Marty Jannetty, Undertaker took on Giant Gonzalez in another terrible match, The Smoking Gunns and Tatanka took on Bam Bam Bigelow and the Headshrinkers, and Yokozuna defended the world title against Lex Luger this match was boring and it has a terrible ending. However it deserves 8/10   \n",
+       "\n",
+       "       label  \n",
+       "33553  1      "
+      ]
+     },
+     "execution_count": 55,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df.head(1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save files\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:20:54.315510Z",
+     "start_time": "2021-04-13T18:20:54.311476Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from joblib import dump"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:20:56.225754Z",
+     "start_time": "2021-04-13T18:20:55.994337Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['../../../data_from_minio/imdb/mapping_rules_labels_t.lib']"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dump(train_rule_matches_sparse_z, os.path.join(data_path, \"train_rule_matches_z.lib\"))\n",
+    "dump(dev_rule_matches_sparse_z, os.path.join(data_path, \"dev_rule_matches_z.lib\"))\n",
+    "dump(test_rule_matches_sparse_z, os.path.join(data_path, \"test_rule_matches_z.lib\"))\n",
+    "\n",
+    "dump(mapping_rules_labels_t,  os.path.join(data_path, \"mapping_rules_labels_t.lib\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We also save the preprocessed texts with labels to use them later to evalutate a classifier (both as csv and binary)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:21:02.417860Z",
+     "start_time": "2021-04-13T18:20:59.438557Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['../../../data_from_minio/imdb/df_test.lib']"
+      ]
+     },
+     "execution_count": 58,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.to_csv(os.path.join(data_path, 'df_train.csv'), index=None)\n",
+    "dev_df.to_csv(os.path.join(data_path, 'df_dev.csv'), index=None)\n",
+    "test_df.to_csv(os.path.join(data_path, 'df_test.csv'), index=None)\n",
+    "\n",
+    "dump(train_df, os.path.join(data_path, \"df_train.lib\"))\n",
+    "dump(dev_df, os.path.join(data_path, \"df_dev.lib\"))\n",
+    "dump(test_df, os.path.join(data_path, \"df_test.lib\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:21:08.061178Z",
+     "start_time": "2021-04-13T18:21:08.040035Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "all_keywords.to_csv(os.path.join(data_path, 'keywords.csv'), index=None)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2021-04-13T18:22:32.546316Z",
+     "start_time": "2021-04-13T18:22:32.539100Z"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['positive-words.txt',\n",
+       " 'dev_rule_matches_z.lib',\n",
+       " 'df_test.csv',\n",
+       " 'keywords.csv',\n",
+       " 'IMDB Dataset.csv',\n",
+       " 'negative-words.txt',\n",
+       " 'df_dev.csv',\n",
+       " 'df_train.csv',\n",
+       " 'df_train.lib',\n",
+       " '.ipynb_checkpoints',\n",
+       " 'test_rule_matches_z.lib',\n",
+       " 'mapping_rules_labels_t.lib',\n",
+       " 'df_dev.lib',\n",
+       " 'train_rule_matches_z.lib',\n",
+       " 'df_test.lib']"
+      ]
+     },
+     "execution_count": 61,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "os.listdir(data_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Finish\n",
+    "\n",
+    "Now, we have created a weak supervision dataset. Of course it is not perfect but it is something with which we can compare performances of different denoising methods with. :-) "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/examples/data_preprocessing/imdb_dataset/__init__.py b/examples/data_preprocessing/imdb_dataset/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/data_preprocessing/imdb_dataset/prepare_weak_imdb_data.py b/examples/data_preprocessing/imdb_dataset/prepare_weak_imdb_data.py
new file mode 100644
index 00000000..978bf341
--- /dev/null
+++ b/examples/data_preprocessing/imdb_dataset/prepare_weak_imdb_data.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+"""
+IMDB Dataset - Create Weak Supervision Sources and Get the Weak Data Annotations
+
+This notebook shows how to use keywords as a weak supervision source on the example of a well-known IMDB Movie Review dataset, which targets a binary sentiment analysis task.
+
+The original dataset has gold labels, but we will use these labels only for evaluation purposes, since we want to test models under the weak supervision setting with Knodle. The idea behind it is that you don't have a dataset which is purely labeled with strong supervision (manual) and instead use heuristics (e.g. rules) to obtain a weak labeling. In the following tutorial, we will look for certain keywords expressing positive and negative sentiments that can be helpful to distinguish between classes. Specifically, we use the [Opinion lexicon](https://www.cs.uic.edu/~liub/FBS/sentiment-analysis.html) of the University of Illinois at Chicago.
+
+First, we load the dataset from a Knodle dataset collection. Then, we will create [Snorkel](https://www.snorkel.org/) labeling functions from two sets of keywords and apply them to the IMDB reviews. Please keep in mind, that keyword matching can be done without Snorkel; however, we enjoy the transparent annotation functionality of this library in our tutorial. 
+Each labeling function (i.e. keyword) will be further associated with a respective target label. This concludes the annotation step.
+
+To estimate how good our weak labeling works on its own, we will use the resulting keyword matches together with a basic majority vote model. Finally, the preprocessed data will be saved in a knodle-friendly format, so that other denoising models can be trained with the IMDB dataset.
+The IMDB dataset available in the Knodle collection was downloaded from [Kaggle](https://www.kaggle.com/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews) in January 2021. 
+"""
+
+import os
+from joblib import dump
+from tqdm import tqdm
+
+import pandas as pd 
+import numpy as np 
+from scipy.sparse import csr_matrix
+
+from bs4 import BeautifulSoup
+from snorkel.labeling import LabelingFunction, PandasLFApplier, LFAnalysis
+
+from knodle.transformation.rule_label_format import transform_snorkel_matrix_to_z_t
+from knodle.transformation.majority import z_t_matrices_to_majority_vote_labels
+
+# client to access the dataset collection
+from minio import Minio
+client = Minio("knodle.dm.univie.ac.at", secure=False)
+
+# init pandas parameters
+tqdm.pandas()
+pd.set_option('display.max_colwidth', -1)
+
+# Constants for Snorkel labeling functions
+POSITIVE = 1
+NEGATIVE = 0
+ABSTAIN = -1
+COLUMN_WITH_TEXT = "reviews_preprocessed"
+
+
+## Download the dataset
+data_path = "../../../data_from_minio/imdb"
+
+# Together with the IMDB data, let us also collect the keywords that we will need later.
+files = [("IMDB Dataset.csv",), ("keywords", "negative-words.txt"), ("keywords", "positive-words.txt")]
+
+for file in tqdm(files):
+    client.fget_object(
+        bucket_name="knodle",
+        object_name=os.path.join("datasets/imdb", *file),
+        file_path=os.path.join(data_path, file[-1]),
+    )
+
+imdb_dataset_raw = pd.read_csv(os.path.join(data_path, "IMDB Dataset.csv"))
+print("Gold label counts")
+print(imdb_dataset_raw.groupby("sentiment").count())
+
+# ## Preprocess dataset
+
+# The dataset contains many HTML tags. We'll remove them
+def strip_html(text):
+    soup = BeautifulSoup(text, "html.parser")
+    return soup.get_text()
+
+imdb_dataset_raw[COLUMN_WITH_TEXT] = imdb_dataset_raw[COLUMN_WITH_TEXT].apply(
+    lambda x: strip_html(x))
+
+# ## Keywords
+# 
+# For weak supervision sources we use sentiment keyword lists for positive and negative words.
+# https://www.cs.uic.edu/~liub/FBS/sentiment-analysis.html
+# 
+# We have downloaded them from the Knodle collection earlier, with the IMDB dataset. 
+# 
+# After parsing the keywords from separate files, they are stored in a pd.DataFrame with the corresponding sentiment as "label".
+
+positive_keywords = pd.read_csv(os.path.join(data_path, "positive-words.txt"), sep=" ", header=None, error_bad_lines=False, skiprows=30)
+positive_keywords.columns = ["keyword"]
+positive_keywords["label"] = "positive"
+
+negative_keywords = pd.read_csv(os.path.join(data_path, "negative-words.txt"),
+                                sep=" ", header=None, error_bad_lines=False,  encoding='latin-1', skiprows=30)
+negative_keywords.columns = ["keyword"]
+negative_keywords["label"] = "negative"
+
+all_keywords = pd.concat([positive_keywords, negative_keywords])
+all_keywords.label.value_counts()
+
+# remove overlap of keywords between two sentiments
+all_keywords.drop_duplicates('keyword', inplace=True)
+
+
+# ## Labeling Functions
+# 
+# Now we start to build labeling functions with Snorkel with these keywords and check the coverage.
+# 
+# This is an iterative process of course so we surely have to add more keywords and regulary expressions ;-) 
+
+def keyword_lookup(x, keyword, label):
+    return label if keyword in x[COLUMN_WITH_TEXT].lower() else ABSTAIN
+
+def make_keyword_lf(keyword: str, label: str) -> LabelingFunction:
+    """
+    Creates labeling function based on keyword.
+    Args:
+        keyword: what keyword should be look for
+        label: what label does this keyword imply
+
+    Returns: LabelingFunction object
+
+    """
+    return LabelingFunction(
+        name=f"keyword_{keyword}",
+        f=keyword_lookup,
+        resources=dict(keyword=keyword, label=label),
+    )
+
+def create_labeling_functions(keywords: pd.DataFrame) -> np.ndarray:
+    """
+    Create Labeling Functions based on the columns keyword and regex. Appends column lf to df.
+
+    Args:
+        keywords: DataFrame with processed keywords
+
+    Returns:
+        All labeling functions. 1d Array with shape: (number_of_lfs x 1)
+    """
+    keywords = keywords.assign(lf=keywords.progress_apply(
+        lambda x:make_keyword_lf(x.keyword, x.label_id), axis=1
+    ))
+    lfs = keywords.lf.values
+    return lfs
+
+all_keywords["label_id"] = all_keywords.label.map({'positive':POSITIVE, 'negative':NEGATIVE})
+labeling_functions = create_labeling_functions(all_keywords)
+
+
+# ### Apply Labeling Functions
+# We get a matrix with all labeling functions applied. This matrix has the shape $(instances \times labeling functions)$
+
+applier = PandasLFApplier(lfs=labeling_functions)
+applied_lfs = applier.apply(df=imdb_dataset_raw)
+
+print("Shape of applied labeling functions: ", applied_lfs.shape)
+print("Number of reviews", len(imdb_dataset_raw))
+print("Number of labeling functions", len(labeling_functions))
+
+
+# ### Analysis 
+# 
+# Now we can analyse some basic stats about our labeling functions. The main figures are:
+# 
+# - Coverage: How many labeling functions match at all
+# - Overlaps: How many labeling functions overlap with each other (e.g. awesome and amazing)
+# - Conflicts: How many labeling functions overlap and have different labels (e.g. awesome and bad)
+# - Correct: Correct LFs
+# - Incorrect: Incorrect Lfs
+
+lf_analysis = LFAnalysis(L=applied_lfs, lfs=labeling_functions).lf_summary()
+print("LF analysis:")
+print(lf_analysis)
+print("LF analysis mean values:")
+print(pd.DataFrame(lf_analysis.mean()))
+print("LF analysis median values:")
+print(pd.DataFrame(lf_analysis.median()))
+
+# Lets have a look at some examples that were labeled by a positive keyword. You can see, that the true label for some of them is negative.
+# consider 1110th keyword
+kw = all_keywords.iloc[1110]
+print(f"1110th keyword is '{kw.keyword}' and labels with '{kw.label}'")
+print()
+
+# sample 2 random examples where the 50th LF assigned a positive label
+examples = imdb_dataset_raw.iloc[applied_lfs[:, 1110] == POSITIVE].loc[:2, [COLUMN_WITH_TEXT,'sentiment']]
+print("This keyword has matched the following instances with labels:")
+print()
+
+for idx, ex in examples.iterrows():
+    print("{} - {}".format(ex.sentiment, ex[COLUMN_WITH_TEXT]))
+    print()
+
+    
+# ## Transform rule matches
+# 
+# To work with knodle the dataset needs to be transformed into a binary array $Z$
+# 
+# (shape: `#instances x # rules_`).
+# 
+# Where a cell $Z_{ij}$ means that for instance i
+# 
+#     0 -> Rule j didn't match 
+#     1 -> Rule j matched
+# 
+# Furthermore, we need a matrix `mapping_rule_labels` which has a mapping of all rules to labels, stored in a binary manner as well 
+# 
+# (shape `#rules x #labels`).
+
+rule_matches, mapping_rules_labels = transform_snorkel_matrix_to_z_t(applied_lfs)
+
+
+# ### Majority Vote
+# 
+# Now we make a majority vote based on all rule matches. First we get the `rule_counts` by multiplying `rule_matches` with the `mapping_rules_labels`, then we divide it sumwise by the sum to get a probability value. In the end we counteract the divide with zero issue by setting all nan values to zero. All this happens in the `z_t_matrices_to_majority_vote_labels` function.
+
+# the ties are resolved randomly internally, so the predictions might slightly vary
+pred_labels = z_t_matrices_to_majority_vote_labels(rule_matches, mapping_rules_labels)
+
+# There are more positive labels predicted by the majority vote.
+labels, counts = np.unique(pred_labels, return_counts=True)
+
+# accuracy of the weak labels
+acc = (pred_labels == imdb_dataset_raw.sentiment.map({'positive':POSITIVE, 'negative':NEGATIVE})).mean()
+print(f"Accuracy of majority voting: {acc}")
+
+      
+# ## Save Files
+rule_matches_sparse = csr_matrix(rule_matches)
+dump(rule_matches_sparse, os.path.join(data_path, "rule_matches.lib"))
+dump(mapping_rules_labels,  os.path.join(data_path, "mapping_rules_labels.lib"))
+
+# We also save the preprocessed texts with labels to use them later to evalutate a classifier.
+imdb_dataset_raw['label_id'] = imdb_dataset_raw.sentiment.map({'positive':POSITIVE, 'negative':NEGATIVE})
+imdb_dataset_raw.to_csv(os.path.join(data_path, 'imdb_data_preprocessed.csv'), index=None)
+
+all_keywords.to_csv(os.path.join(data_path, 'keywords.csv'), index=None)
+
+# Now, we have created a weak supervision dataset. Of course it is not perfect but it is something with which we can compare performances of different denoising methods with. :-) 
diff --git a/examples/data_preprocessing/imdb_dataset/requirements.txt b/examples/data_preprocessing/imdb_dataset/requirements.txt
new file mode 100644
index 00000000..4b1987f7
--- /dev/null
+++ b/examples/data_preprocessing/imdb_dataset/requirements.txt
@@ -0,0 +1,9 @@
+minio
+cython
+pandas
+numpy
+jupyter
+snorkel
+bs4
+matplotlib
+tqdm
diff --git a/examples/data_preprocessing/tac_based_dataset/[WIP]_patterns_aka_lfs/Prepare_TAC_based_RE_dataset_with_patters_aka_lfs.ipynb b/examples/data_preprocessing/tac_based_dataset/[WIP]_patterns_aka_lfs/Prepare_TAC_based_RE_dataset_with_patters_aka_lfs.ipynb
new file mode 100644
index 00000000..a50590f6
--- /dev/null
+++ b/examples/data_preprocessing/tac_based_dataset/[WIP]_patterns_aka_lfs/Prepare_TAC_based_RE_dataset_with_patters_aka_lfs.ipynb
@@ -0,0 +1,1728 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Relation Exraction Dataset based on CONLL data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this small tutorial we are going to look at how we could make the weak annotation of data for relation extraction task using simple patterns. \n",
+    "\n",
+    "For that, we are going to use dataset in CONLL format that already has gold labels, which we are going to use for comparing with the labels we receive. The examples of such files could be downloaded by the following link: https://ucloud.univie.ac.at/index.php/s/cGQNRgbPHZU89mY\n",
+    "\n",
+    "The steps are the following:\n",
+    "- process the data in CONLL format (an example could be downloaded by the following link:)\n",
+    "- extract all the relevant for us information (the sample and the relation label)\n",
+    "- create a sample set of hand-crafted patterns in the format: _\"\\\\$ARG1 \\<some words\\> \\\\$ARG2\"_ where _\\\\$ARG1_ and _\\\\$ARG2_ are the entities between that the relation is hold\n",
+    "- annotated sample with Spacy package in order to get the entities\n",
+    "- take the entities pairwise and extract the part of sentences between them \n",
+    "\n",
+    "(e.g. from a sentence _\"Margaret Thatcher was born on 13 October 1925\"_ a subsentence _\"\\\\$ARG1 was born on \\\\$ARG2\"_ will be extracted given _\"Margaret Thatcher\"_ and _\"13 October 1925\"_ were the only entities defined by Spacy)\n",
+    "\n",
+    "- check if there is a pattern in this subsentences. If so, it receives a label corresponding to the relation that pattern support and becomes a part of our new weakly annotated training set. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/anaconda3/lib/python3.8/site-packages/tqdm/std.py:668: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
+      "  from pandas import Panel\n"
+     ]
+    }
+   ],
+   "source": [
+    "import spacy\n",
+    "import pandas as pd\n",
+    "# import en_core_web_sm\n",
+    "import re\n",
+    "import itertools\n",
+    "from tqdm import tqdm\n",
+    "tqdm.pandas()\n",
+    "\n",
+    "pd.set_option('display.max_colwidth', -1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 79,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# in oder to avoid issues with en_core_web_lg, please execute the following command\n",
+    "!python -m spacy download en_core_web_sm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "ARG1 = \"$ARG1\"\n",
+    "ARG2 = \"$ARG2\"\n",
+    "FINAL_DF_COLUMNS = ['sample', 'extr_sample', 'pattern', 'weak_label', 'gold_label']"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We are going to use this function each time we want to print out a Dataframe in order to escape dollar symbol in patterns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 80,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def escape_dollar(strings):\n",
+    "    return [re.sub(\"\\\\$\", \"\\\\\\\\$\", str(string)) for string in strings]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Read and preprocess CONLL data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Firstly, we are going to read the input .conll file. Please make sure that it is saved in the same directory as this tutorial notebook and path to it is solved in conll_file variable."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 81,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "conll_file = \"dev.conll\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 82,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def process_data(path_to_data):\n",
+    "    samples, relations = [], []\n",
+    "    with open(path_to_data, encoding='utf-8') as f:\n",
+    "        for line in f:\n",
+    "            line = line.strip()\n",
+    "            if line.startswith(\"# id=\"):    # Instance starts\n",
+    "                sample = \"\"\n",
+    "                label = line.split(\" \")[3][5:]\n",
+    "            elif line == \"\":  # Instance ends\n",
+    "                samples.append(sample)\n",
+    "                relations.append(label)\n",
+    "            elif line.startswith(\"#\"):  # comment\n",
+    "                continue\n",
+    "            else:\n",
+    "                parts = line.split(\"\\t\")\n",
+    "                token = parts[1]\n",
+    "                if token == \"-LRB-\":\n",
+    "                    token = \"(\"\n",
+    "                elif token == \"-RRB-\":\n",
+    "                    token = \")\"\n",
+    "                sample += \" \" + token\n",
+    "    return pd.DataFrame.from_dict({\"sample\": samples, \"label\": relations})\n",
+    "\n",
+    "samples = process_data(conll_file)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's inspect all relation labels that we have in our dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 83,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'org:founded', 'org:country_of_headquarters', 'per:city_of_birth', 'per:origin', 'per:other_family', 'per:country_of_birth', 'org:dissolved', 'org:shareholders', 'per:parents', 'per:cause_of_death', 'per:spouse', 'per:stateorprovince_of_death', 'no_relation', 'org:members', 'org:stateorprovince_of_headquarters', 'org:number_of_employees/members', 'per:cities_of_residence', 'org:political/religious_affiliation', 'per:children', 'per:date_of_death', 'org:top_members/employees', 'org:city_of_headquarters', 'org:subsidiaries', 'per:city_of_death', 'per:countries_of_residence', 'per:schools_attended', 'per:stateorprovinces_of_residence', 'org:alternate_names', 'org:parents', 'per:religion', 'org:founded_by', 'per:age', 'per:employee_of', 'per:title', 'per:date_of_birth', 'org:member_of', 'per:alternate_names', 'per:country_of_death', 'per:stateorprovince_of_birth', 'per:charges', 'org:website', 'per:siblings'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(set(samples[\"label\"]))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To make the calculation quicklier, let's choose only the samples that contain a relation (that is, labelled not with \"no_relation\" label). "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 84,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "selected_samples = samples[samples[\"label\"]!='no_relation'].sample(n=1000, random_state=100)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 85,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sample</th>\n",
+       "      <th>label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>4777</th>\n",
+       "      <td>Or as one hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their Priority Club frequency points , too . ''</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4068</th>\n",
+       "      <td>Prolific French film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told AFP .</td>\n",
+       "      <td>per:origin</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5581</th>\n",
+       "      <td>Try the Public Library of Science ( http://www.plos.org/ ) , which does have some drawbacks , but also appears to have a clue .</td>\n",
+       "      <td>org:website</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21851</th>\n",
+       "      <td>His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .</td>\n",
+       "      <td>per:religion</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4237</th>\n",
+       "      <td>Popular Kabul lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , Najafi added .</td>\n",
+       "      <td>per:cities_of_residence</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                            sample  \\\n",
+       "4777    Or as one hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their Priority Club frequency points , too . ''                                         \n",
+       "4068    Prolific French film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told AFP .                                              \n",
+       "5581    Try the Public Library of Science ( http://www.plos.org/ ) , which does have some drawbacks , but also appears to have a clue .                                                                                              \n",
+       "21851   His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .              \n",
+       "4237    Popular Kabul lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , Najafi added .   \n",
+       "\n",
+       "                           label  \n",
+       "4777   org:top_members/employees  \n",
+       "4068   per:origin                 \n",
+       "5581   org:website                \n",
+       "21851  per:religion               \n",
+       "4237   per:cities_of_residence    "
+      ]
+     },
+     "execution_count": 85,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "selected_samples.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Defining patterns"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In order to turn the data into distantly supervised one, we write down a couple of simple patterns for each relation that could help us to find these relations in training samples. For complexity reasons we reduced the number of relation we want to write patterns for and choose 3 relations from the TACRED relation labels: \"org:alternate_names\", \"per:date_of_birth\" and \"org:top_members/employees\". "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 86,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "relation_patterns = {\"org:alternate_names\":\n",
+    "                     [\"$ARG1 ( $ARG2 \", \n",
+    "                       \"$ARG1 formerly known as $ARG2\",\n",
+    "                       \"$ARG1 aka $ARG2\", \n",
+    "                       \"$ARG1 ( also known as $ARG2 )\"],\n",
+    "                     \"per:date_of_birth\": \n",
+    "                     [\"$ARG1 ( born $ARG2 )\", \n",
+    "                       \"$ARG1 ( born $ARG2 in\",\n",
+    "                       \"$ARG1 ( $ARG2 -\",\n",
+    "                       \"$ARG1 was born in $ARG2\"],\n",
+    "                     \"org:top_members/employees\":\n",
+    "                     [\"$ARG1 , executive director of $ARG2\",\n",
+    "                       \"$ARG1 , head of $ARG2\",\n",
+    "                       \"$ARG1 , who heads $ARG2\",\n",
+    "                       \"$ARG1 , chief executive of $ARG2\"]}\n",
+    "\n",
+    "relation_patterns_df = pd.DataFrame.from_dict(relation_patterns, orient='index')\n",
+    "relation_patterns_df['raw pattern']=relation_patterns_df.apply(lambda row: row.dropna().tolist(), axis=1)\n",
+    "relation_patterns_df = relation_patterns_df[['raw pattern']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 87,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>raw pattern</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>org:alternate_names</th>\n",
+       "      <td>['\\$ARG1 ( \\$ARG2 ', '\\$ARG1 formerly known as \\$ARG2', '\\$ARG1 aka \\$ARG2', '\\$ARG1 ( also known as \\$ARG2 )']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>per:date_of_birth</th>\n",
+       "      <td>['\\$ARG1 ( born \\$ARG2 )', '\\$ARG1 ( born \\$ARG2 in', '\\$ARG1 ( \\$ARG2 -', '\\$ARG1 was born in \\$ARG2']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>org:top_members/employees</th>\n",
+       "      <td>['\\$ARG1 , executive director of \\$ARG2', '\\$ARG1 , head of \\$ARG2', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , chief executive of \\$ARG2']</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                       raw pattern\n",
+       "org:alternate_names        ['\\$ARG1 ( \\$ARG2 ', '\\$ARG1 formerly known as \\$ARG2', '\\$ARG1 aka \\$ARG2', '\\$ARG1 ( also known as \\$ARG2 )']                        \n",
+       "per:date_of_birth          ['\\$ARG1 ( born \\$ARG2 )', '\\$ARG1 ( born \\$ARG2 in', '\\$ARG1 ( \\$ARG2 -', '\\$ARG1 was born in \\$ARG2']                                \n",
+       "org:top_members/employees  ['\\$ARG1 , executive director of \\$ARG2', '\\$ARG1 , head of \\$ARG2', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , chief executive of \\$ARG2']"
+      ]
+     },
+     "execution_count": 87,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "relation_patterns_df.apply(lambda x: escape_dollar(x)).head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "Since we want to make a simple regex search, we convert the handßwritten patterns into regexes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 88,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def preprocess_patterns(patterns):\n",
+    "    regex_patterns = [re.sub(\"\\\\\\\\\\\\$ARG\", \"(A )?(a )?(The )?(the )?\\\\$ARG\", re.escape(pattern)) for pattern in patterns]\n",
+    "    return regex_patterns\n",
+    "\n",
+    "relation_patterns_df[\"regex pattern\"] = relation_patterns_df[\"raw pattern\"].apply(preprocess_patterns)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 89,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>raw pattern</th>\n",
+       "      <th>regex pattern</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>org:alternate_names</th>\n",
+       "      <td>['\\$ARG1 ( \\$ARG2 ', '\\$ARG1 formerly known as \\$ARG2', '\\$ARG1 aka \\$ARG2', '\\$ARG1 ( also known as \\$ARG2 )']</td>\n",
+       "      <td>['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ ', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ formerly\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ aka\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ also\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>per:date_of_birth</th>\n",
+       "      <td>['\\$ARG1 ( born \\$ARG2 )', '\\$ARG1 ( born \\$ARG2 in', '\\$ARG1 ( \\$ARG2 -', '\\$ARG1 was born in \\$ARG2']</td>\n",
+       "      <td>['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ in', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\-', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ was\\\\ born\\\\ in\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>org:top_members/employees</th>\n",
+       "      <td>['\\$ARG1 , executive director of \\$ARG2', '\\$ARG1 , head of \\$ARG2', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , chief executive of \\$ARG2']</td>\n",
+       "      <td>['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ executive\\\\ director\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ head\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ who\\\\ heads\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ chief\\\\ executive\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                       raw pattern  \\\n",
+       "org:alternate_names        ['\\$ARG1 ( \\$ARG2 ', '\\$ARG1 formerly known as \\$ARG2', '\\$ARG1 aka \\$ARG2', '\\$ARG1 ( also known as \\$ARG2 )']                           \n",
+       "per:date_of_birth          ['\\$ARG1 ( born \\$ARG2 )', '\\$ARG1 ( born \\$ARG2 in', '\\$ARG1 ( \\$ARG2 -', '\\$ARG1 was born in \\$ARG2']                                   \n",
+       "org:top_members/employees  ['\\$ARG1 , executive director of \\$ARG2', '\\$ARG1 , head of \\$ARG2', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , chief executive of \\$ARG2']   \n",
+       "\n",
+       "                                                                                                                                                                                                                                                                                                                                                                                                         regex pattern  \n",
+       "org:alternate_names        ['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ ', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ formerly\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ aka\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ also\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)']                          \n",
+       "per:date_of_birth          ['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ in', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\-', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ was\\\\ born\\\\ in\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']                              \n",
+       "org:top_members/employees  ['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ executive\\\\ director\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ head\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ who\\\\ heads\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ chief\\\\ executive\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']  "
+      ]
+     },
+     "execution_count": 89,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "relation_patterns_df.apply(lambda x: escape_dollar(x)).head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Entity pairing"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We get the sentence annotations with the Spacy package"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 90,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "analyzer = spacy.load(\"en_core_web_sm\")\n",
+    "selected_samples[\"spacy info\"] = selected_samples[\"sample\"].apply(lambda x: analyzer(x).to_json())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "After that, we take the arguments pairwise in each sentence and create subsentences that include:\n",
+    "- the first entity substituted with \"ARG1\"\n",
+    "- the second argument substituted with \"$ARG2\"\n",
+    "- all the words inbetween"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def get_extracted_sample(sample):\n",
+    "    return [(ARG1 + sample[\"text\"][ent1[\"end\"]:ent2[\"start\"]] + ARG2) if ent1[\"end\"] < ent2[\"end\"] \n",
+    "            else (ARG2 + sample[\"text\"][ent2[\"end\"]:ent1[\"start\"]] + ARG1)\n",
+    "            for ent1, ent2 in itertools.permutations(sample[\"ents\"],2)]\n",
+    "\n",
+    "selected_samples[\"extr\"] = selected_samples[\"spacy info\"].apply(lambda x: get_extracted_sample(x))\n",
+    "selected_samples = selected_samples.loc[selected_samples[\"extr\"].apply(lambda x: len(x) != 0)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 92,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sample</th>\n",
+       "      <th>label</th>\n",
+       "      <th>extr</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>4777</th>\n",
+       "      <td>Or as one hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their Priority Club frequency points , too . ''</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>['\\$ARG1 hotel executive , \\$ARG2', '\\$ARG1 hotel executive , James Anhut , of \\$ARG2', '\\$ARG1 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` \\$ARG2', '\\$ARG1 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , \\$ARG1', '\\$ARG1 , of \\$ARG2', '\\$ARG1 , of InterContinental Hotels Group , put it , `` \\$ARG2', '\\$ARG1 , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , James Anhut , of \\$ARG1', '\\$ARG2 , of \\$ARG1', '\\$ARG1 , put it , `` \\$ARG2', '\\$ARG1 , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` \\$ARG1', '\\$ARG2 , of InterContinental Hotels Group , put it , `` \\$ARG1', '\\$ARG2 , put it , `` \\$ARG1', '\\$ARG1 can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG1', '\\$ARG2 , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG1', '\\$ARG2 , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG1', '\\$ARG2 can stay in a cool hotel and earn their \\$ARG1']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4068</th>\n",
+       "      <td>Prolific French film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told AFP .</td>\n",
+       "      <td>per:origin</td>\n",
+       "      <td>['\\$ARG1 film maker \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker \\$ARG1', '\\$ARG1 , who helped start the \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the \\$ARG1', '\\$ARG2 , who helped start the \\$ARG1', '\\$ARG1 movement in \\$ARG2', '\\$ARG1 movement in the 1950s , died on \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in \\$ARG1', '\\$ARG2 movement in \\$ARG1', '\\$ARG1 , died on \\$ARG2', '\\$ARG1 , died on Sunday , aged \\$ARG2', '\\$ARG1 , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on \\$ARG1', '\\$ARG2 movement in the 1950s , died on \\$ARG1', '\\$ARG2 , died on \\$ARG1', '\\$ARG1 , aged \\$ARG2', '\\$ARG1 , aged 80 , an aide to \\$ARG2', '\\$ARG1 , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged \\$ARG1', '\\$ARG2 , died on Sunday , aged \\$ARG1', '\\$ARG2 , aged \\$ARG1', '\\$ARG1 , an aide to \\$ARG2', '\\$ARG1 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 , aged 80 , an aide to \\$ARG1', '\\$ARG2 , an aide to \\$ARG1', '\\$ARG1 mayor \\$ARG2', '\\$ARG1 mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , an aide to Paris mayor \\$ARG1', '\\$ARG2 mayor \\$ARG1', '\\$ARG1 told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 told \\$ARG1']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21851</th>\n",
+       "      <td>His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .</td>\n",
+       "      <td>per:religion</td>\n",
+       "      <td>['\\$ARG1 rival , \\$ARG2', '\\$ARG1 rival , Abdul Aziz al-Hakim , who heads \\$ARG2', '\\$ARG1 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG2', '\\$ARG1 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG2', '\\$ARG2 rival , \\$ARG1', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG2', '\\$ARG1 , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG2', '\\$ARG2 rival , Abdul Aziz al-Hakim , who heads \\$ARG1', '\\$ARG2 , who heads \\$ARG1', '\\$ARG1 , an influential \\$ARG2', '\\$ARG1 , an influential Shiite political party that is part of \\$ARG2', '\\$ARG2 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG1', '\\$ARG2 , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG1', '\\$ARG2 , an influential \\$ARG1', '\\$ARG1 political party that is part of \\$ARG2', '\\$ARG2 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG1', '\\$ARG2 , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG1', '\\$ARG2 , an influential Shiite political party that is part of \\$ARG1', '\\$ARG2 political party that is part of \\$ARG1']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4237</th>\n",
+       "      <td>Popular Kabul lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , Najafi added .</td>\n",
+       "      <td>per:cities_of_residence</td>\n",
+       "      <td>['\\$ARG1 lawmaker \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker \\$ARG1', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG1', '\\$ARG1 votes and former \\$ARG2', '\\$ARG1 votes and former World Bank economist \\$ARG2', '\\$ARG1 votes and former World Bank economist Ashraf Ghani \\$ARG2', '\\$ARG1 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG1', '\\$ARG2 votes and former \\$ARG1', '\\$ARG1 economist \\$ARG2', '\\$ARG1 economist Ashraf Ghani \\$ARG2', '\\$ARG1 economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG1', '\\$ARG2 votes and former World Bank economist \\$ARG1', '\\$ARG2 economist \\$ARG1', '\\$ARG1 \\$ARG2', '\\$ARG1 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG1', '\\$ARG2 votes and former World Bank economist Ashraf Ghani \\$ARG1', '\\$ARG2 economist Ashraf Ghani \\$ARG1', '\\$ARG2 \\$ARG1', '\\$ARG1 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 62,536 votes , \\$ARG1', '\\$ARG2 votes , \\$ARG1']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3311</th>\n",
+       "      <td>`` Our entire organization grieves at the death of Mike Coolbaugh , '' Rockies president Keli McGregor said .</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>[\"\\$ARG1 , '' Rockies president \\$ARG2\", \"\\$ARG2 , '' Rockies president \\$ARG1\"]</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                            sample  \\\n",
+       "4777    Or as one hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their Priority Club frequency points , too . ''                                         \n",
+       "4068    Prolific French film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told AFP .                                              \n",
+       "21851   His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .              \n",
+       "4237    Popular Kabul lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , Najafi added .   \n",
+       "3311    `` Our entire organization grieves at the death of Mike Coolbaugh , '' Rockies president Keli McGregor said .                                                                                                                \n",
+       "\n",
+       "                           label  \\\n",
+       "4777   org:top_members/employees   \n",
+       "4068   per:origin                  \n",
+       "21851  per:religion                \n",
+       "4237   per:cities_of_residence     \n",
+       "3311   per:title                   \n",
+       "\n",
+       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 extr  \n",
+       "4777   ['\\$ARG1 hotel executive , \\$ARG2', '\\$ARG1 hotel executive , James Anhut , of \\$ARG2', '\\$ARG1 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` \\$ARG2', '\\$ARG1 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , \\$ARG1', '\\$ARG1 , of \\$ARG2', '\\$ARG1 , of InterContinental Hotels Group , put it , `` \\$ARG2', '\\$ARG1 , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , James Anhut , of \\$ARG1', '\\$ARG2 , of \\$ARG1', '\\$ARG1 , put it , `` \\$ARG2', '\\$ARG1 , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` \\$ARG1', '\\$ARG2 , of InterContinental Hotels Group , put it , `` \\$ARG1', '\\$ARG2 , put it , `` \\$ARG1', '\\$ARG1 can stay in a cool hotel and earn their \\$ARG2', '\\$ARG2 hotel executive , James Anhut , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG1', '\\$ARG2 , of InterContinental Hotels Group , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG1', '\\$ARG2 , put it , `` Travelers can stay in a cool hotel and earn their \\$ARG1', '\\$ARG2 can stay in a cool hotel and earn their \\$ARG1']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
+       "4068   ['\\$ARG1 film maker \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker \\$ARG1', '\\$ARG1 , who helped start the \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the \\$ARG1', '\\$ARG2 , who helped start the \\$ARG1', '\\$ARG1 movement in \\$ARG2', '\\$ARG1 movement in the 1950s , died on \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in \\$ARG1', '\\$ARG2 movement in \\$ARG1', '\\$ARG1 , died on \\$ARG2', '\\$ARG1 , died on Sunday , aged \\$ARG2', '\\$ARG1 , died on Sunday , aged 80 , an aide to \\$ARG2', '\\$ARG1 , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on \\$ARG1', '\\$ARG2 movement in the 1950s , died on \\$ARG1', '\\$ARG2 , died on \\$ARG1', '\\$ARG1 , aged \\$ARG2', '\\$ARG1 , aged 80 , an aide to \\$ARG2', '\\$ARG1 , aged 80 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged \\$ARG1', '\\$ARG2 , died on Sunday , aged \\$ARG1', '\\$ARG2 , aged \\$ARG1', '\\$ARG1 , an aide to \\$ARG2', '\\$ARG1 , an aide to Paris mayor \\$ARG2', '\\$ARG1 , an aide to Paris mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 , died on Sunday , aged 80 , an aide to \\$ARG1', '\\$ARG2 , aged 80 , an aide to \\$ARG1', '\\$ARG2 , an aide to \\$ARG1', '\\$ARG1 mayor \\$ARG2', '\\$ARG1 mayor Bertrand Delanoe told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , died on Sunday , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , aged 80 , an aide to Paris mayor \\$ARG1', '\\$ARG2 , an aide to Paris mayor \\$ARG1', '\\$ARG2 mayor \\$ARG1', '\\$ARG1 told \\$ARG2', '\\$ARG2 film maker Claude Chabrol , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , who helped start the New Wave movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 movement in the 1950s , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , died on Sunday , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , aged 80 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 , an aide to Paris mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 mayor Bertrand Delanoe told \\$ARG1', '\\$ARG2 told \\$ARG1']  \n",
+       "21851  ['\\$ARG1 rival , \\$ARG2', '\\$ARG1 rival , Abdul Aziz al-Hakim , who heads \\$ARG2', '\\$ARG1 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG2', '\\$ARG1 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG2', '\\$ARG2 rival , \\$ARG1', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG2', '\\$ARG1 , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG2', '\\$ARG2 rival , Abdul Aziz al-Hakim , who heads \\$ARG1', '\\$ARG2 , who heads \\$ARG1', '\\$ARG1 , an influential \\$ARG2', '\\$ARG1 , an influential Shiite political party that is part of \\$ARG2', '\\$ARG2 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG1', '\\$ARG2 , who heads the Islamic Supreme Council of Iraq , an influential \\$ARG1', '\\$ARG2 , an influential \\$ARG1', '\\$ARG1 political party that is part of \\$ARG2', '\\$ARG2 rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG1', '\\$ARG2 , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of \\$ARG1', '\\$ARG2 , an influential Shiite political party that is part of \\$ARG1', '\\$ARG2 political party that is part of \\$ARG1']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
+       "4237   ['\\$ARG1 lawmaker \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG2', '\\$ARG1 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker \\$ARG1', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG2', '\\$ARG1 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted \\$ARG1', '\\$ARG1 votes and former \\$ARG2', '\\$ARG1 votes and former World Bank economist \\$ARG2', '\\$ARG1 votes and former World Bank economist Ashraf Ghani \\$ARG2', '\\$ARG1 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former \\$ARG1', '\\$ARG2 votes and former \\$ARG1', '\\$ARG1 economist \\$ARG2', '\\$ARG1 economist Ashraf Ghani \\$ARG2', '\\$ARG1 economist Ashraf Ghani 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist \\$ARG1', '\\$ARG2 votes and former World Bank economist \\$ARG1', '\\$ARG2 economist \\$ARG1', '\\$ARG1 \\$ARG2', '\\$ARG1 62,536 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani \\$ARG1', '\\$ARG2 votes and former World Bank economist Ashraf Ghani \\$ARG1', '\\$ARG2 economist Ashraf Ghani \\$ARG1', '\\$ARG2 \\$ARG1', '\\$ARG1 votes , \\$ARG2', '\\$ARG2 lawmaker Ramazan Bashardost , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 , who camps out in a tent near parliament and campaigned against corruption , attracted 359,214 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 votes and former World Bank economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 economist Ashraf Ghani 62,536 votes , \\$ARG1', '\\$ARG2 62,536 votes , \\$ARG1', '\\$ARG2 votes , \\$ARG1']                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \n",
+       "3311   [\"\\$ARG1 , '' Rockies president \\$ARG2\", \"\\$ARG2 , '' Rockies president \\$ARG1\"]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                "
+      ]
+     },
+     "execution_count": 92,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "selected_samples[[\"sample\", \"label\", \"extr\"]].apply(lambda x: escape_dollar(x)).head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Pattern search"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we look for patterns in each of these subsentences. If there is a match, we add a corresponding sentence to the final DataFrame with the information about pattern that matched and weak label it received. The fold labels are still there for comparing. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 93,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def pattern_search(extr_sample, patterns, row):\n",
+    "    for relation, rel_patterns in patterns.iterrows():\n",
+    "        matches = [[row[\"sample\"], extr_sample, pattern, relation, row[\"label\"]] \n",
+    "                   for pattern in rel_patterns[\"regex pattern\"]\n",
+    "                   if re.match(pattern, extr_sample) is not None]   # todo\n",
+    "        if len(matches) > 0:\n",
+    "            return pd.DataFrame(matches, columns = FINAL_DF_COLUMNS)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "all_matches = pd.DataFrame(columns = FINAL_DF_COLUMNS)\n",
+    "for _, row in selected_samples.iterrows():\n",
+    "    for cand_sample in row[\"extr\"]:\n",
+    "        df_found = pattern_search(cand_sample, relation_patterns_df, row)\n",
+    "        if isinstance(df_found, pd.DataFrame) and not df_found.empty:\n",
+    "            all_matches = pd.concat([all_matches, df_found])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 95,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sample</th>\n",
+       "      <th>extr_sample</th>\n",
+       "      <th>pattern</th>\n",
+       "      <th>weak_label</th>\n",
+       "      <th>gold_label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .</td>\n",
+       "      <td>\\$ARG1 , who heads \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ who\\ heads\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>per:religion</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>`` It 's an issue for everybody in the state because peanuts are a big part of our economy , '' said Don Koehler , executive director of the Georgia Peanut Commission .</td>\n",
+       "      <td>\\$ARG1 , executive director of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .</td>\n",
+       "      <td>\\$ARG1 , executive director of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .</td>\n",
+       "      <td>\\$ARG1 was born in \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>per:date_of_birth</td>\n",
+       "      <td>per:children</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>A professor emeritus at Yale University , Mandelbrot was born in Poland but as a child moved with his family to France where he was educated .</td>\n",
+       "      <td>\\$ARG1 was born in \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>per:date_of_birth</td>\n",
+       "      <td>per:employee_of</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                      sample  \\\n",
+       "0   His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .            \n",
+       "0   `` It 's an issue for everybody in the state because peanuts are a big part of our economy , '' said Don Koehler , executive director of the Georgia Peanut Commission .                                                   \n",
+       "0   Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .   \n",
+       "0   Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .                                                                                                            \n",
+       "0   A professor emeritus at Yale University , Mandelbrot was born in Poland but as a child moved with his family to France where he was educated .                                                                             \n",
+       "\n",
+       "                             extr_sample  \\\n",
+       "0  \\$ARG1 , who heads \\$ARG2               \n",
+       "0  \\$ARG1 , executive director of \\$ARG2   \n",
+       "0  \\$ARG1 , executive director of \\$ARG2   \n",
+       "0  \\$ARG1 was born in \\$ARG2               \n",
+       "0  \\$ARG1 was born in \\$ARG2               \n",
+       "\n",
+       "                                                                                        pattern  \\\n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ who\\ heads\\ (A )?(a )?(The )?(the )?\\\\$ARG2                \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2   \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2   \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2                \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2                \n",
+       "\n",
+       "                  weak_label                 gold_label  \n",
+       "0  org:top_members/employees  per:religion               \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  org:top_members/employees  org:alternate_names        \n",
+       "0  per:date_of_birth          per:children               \n",
+       "0  per:date_of_birth          per:employee_of            "
+      ]
+     },
+     "execution_count": 95,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_matches.apply(lambda x: escape_dollar(x)).head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 96,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "105"
+      ]
+     },
+     "execution_count": 96,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_matches.size"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "But here we can observe some misclassified sentences: for example, a sentence \n",
+    "\n",
+    "\"A professor emeritus at Yale University , Mandelbrot was born in Poland but as a child moved with his family to France where he was educated\" \n",
+    "\n",
+    "was assigned with a label \"per:date_of_birth\" (presumably by a pattern \"ARG1 was born in ARG2\"), what is definitely wrong. In order to avoid such mistake, let's add additional constraints on the argument types."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 97,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "relation_to_types = {\"org:alternate_names\": ['PERSON', 'PERSON'], \n",
+    "                     \"per:date_of_birth\": ['PERSON', 'DATE'],\n",
+    "                     \"org:top_members/employees\": ['PERSON', 'ORG']}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "So, when we look for these patterns in samples, we should take into account the entity types referred to corresponding relation. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 98,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "relation_patterns_df[\"type\"] = pd.Series(relation_to_types)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 99,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>raw pattern</th>\n",
+       "      <th>regex pattern</th>\n",
+       "      <th>type</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>org:alternate_names</th>\n",
+       "      <td>['\\$ARG1 ( \\$ARG2 ', '\\$ARG1 formerly known as \\$ARG2', '\\$ARG1 aka \\$ARG2', '\\$ARG1 ( also known as \\$ARG2 )']</td>\n",
+       "      <td>['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ ', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ formerly\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ aka\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ also\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)']</td>\n",
+       "      <td>['PERSON', 'PERSON']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>per:date_of_birth</th>\n",
+       "      <td>['\\$ARG1 ( born \\$ARG2 )', '\\$ARG1 ( born \\$ARG2 in', '\\$ARG1 ( \\$ARG2 -', '\\$ARG1 was born in \\$ARG2']</td>\n",
+       "      <td>['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ in', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\-', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ was\\\\ born\\\\ in\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']</td>\n",
+       "      <td>['PERSON', 'DATE']</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>org:top_members/employees</th>\n",
+       "      <td>['\\$ARG1 , executive director of \\$ARG2', '\\$ARG1 , head of \\$ARG2', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , chief executive of \\$ARG2']</td>\n",
+       "      <td>['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ executive\\\\ director\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ head\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ who\\\\ heads\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ chief\\\\ executive\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']</td>\n",
+       "      <td>['PERSON', 'ORG']</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                       raw pattern  \\\n",
+       "org:alternate_names        ['\\$ARG1 ( \\$ARG2 ', '\\$ARG1 formerly known as \\$ARG2', '\\$ARG1 aka \\$ARG2', '\\$ARG1 ( also known as \\$ARG2 )']                           \n",
+       "per:date_of_birth          ['\\$ARG1 ( born \\$ARG2 )', '\\$ARG1 ( born \\$ARG2 in', '\\$ARG1 ( \\$ARG2 -', '\\$ARG1 was born in \\$ARG2']                                   \n",
+       "org:top_members/employees  ['\\$ARG1 , executive director of \\$ARG2', '\\$ARG1 , head of \\$ARG2', '\\$ARG1 , who heads \\$ARG2', '\\$ARG1 , chief executive of \\$ARG2']   \n",
+       "\n",
+       "                                                                                                                                                                                                                                                                                                                                                                                                         regex pattern  \\\n",
+       "org:alternate_names        ['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ ', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ formerly\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ aka\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ also\\\\ known\\\\ as\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)']                           \n",
+       "per:date_of_birth          ['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\)', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ born\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ in', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ \\\\(\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2\\\\ \\\\-', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ was\\\\ born\\\\ in\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']                               \n",
+       "org:top_members/employees  ['(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ executive\\\\ director\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ head\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ who\\\\ heads\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2', '(A )?(a )?(The )?(the )?\\\\\\$ARG1\\\\ ,\\\\ chief\\\\ executive\\\\ of\\\\ (A )?(a )?(The )?(the )?\\\\\\$ARG2']   \n",
+       "\n",
+       "                                           type  \n",
+       "org:alternate_names        ['PERSON', 'PERSON']  \n",
+       "per:date_of_birth          ['PERSON', 'DATE']    \n",
+       "org:top_members/employees  ['PERSON', 'ORG']     "
+      ]
+     },
+     "execution_count": 99,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "relation_patterns_df.apply(lambda x: escape_dollar(x)).head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The information about type of the entities we get from the Spacy annotations as well and build a new format of extracted subsentences: now they are not lists but dictionaries in format {subsentence: entity types}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 100,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_arg_types(sample):\n",
+    "    return [{ARG1 + sample[\"text\"][ent1[\"end\"]:ent2[\"start\"]] + ARG2 : [ent1[\"label\"], ent2[\"label\"]]}\n",
+    "            if ent1[\"end\"] < ent2[\"end\"]\n",
+    "            else {(ARG2 + sample[\"text\"][ent2[\"end\"]:ent1[\"start\"]] + ARG1) : [ent2[\"label\"], ent1[\"label\"]]}\n",
+    "            for ent1, ent2 in itertools.permutations(sample[\"ents\"],2)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 101,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "selected_samples[\"extr_with_types\"] = selected_samples[\"spacy info\"].apply(lambda x: get_arg_types(x))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "We do a similar pattern search, but now taking into account entity types."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 102,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def pattern_search_filter_types(cand_sample_with_types, patterns, row):\n",
+    "    for relation, rel_patterns in patterns.iterrows():\n",
+    "        cand_sample_types = [item for type_pair in cand_sample_with_types.values() for item in type_pair]\n",
+    "        if rel_patterns[\"type\"] == cand_sample_types:\n",
+    "            sample = list(cand_sample_with_types.keys())[0]\n",
+    "            matches = [[row[\"sample\"], sample, pattern, relation, row[\"label\"]] \n",
+    "                       for pattern in rel_patterns[\"regex pattern\"]\n",
+    "                       if re.match(pattern, sample) is not None]   # todo\n",
+    "            if len(matches) > 0:\n",
+    "                return pd.DataFrame(matches, columns = FINAL_DF_COLUMNS)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 103,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_matches_filter_types = pd.DataFrame(columns = FINAL_DF_COLUMNS)\n",
+    "for _, row in selected_samples.iterrows():\n",
+    "    for cand_sample_with_types in row[\"extr_with_types\"]:\n",
+    "        df_found = pattern_search_filter_types(cand_sample_with_types, relation_patterns_df, row)\n",
+    "        if isinstance(df_found, pd.DataFrame) and not df_found.empty:\n",
+    "            all_matches_filter_types = pd.concat([all_matches_filter_types, df_found])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 104,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sample</th>\n",
+       "      <th>extr_sample</th>\n",
+       "      <th>pattern</th>\n",
+       "      <th>weak_label</th>\n",
+       "      <th>gold_label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .</td>\n",
+       "      <td>\\$ARG1 , who heads \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ who\\ heads\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>per:religion</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>`` It 's an issue for everybody in the state because peanuts are a big part of our economy , '' said Don Koehler , executive director of the Georgia Peanut Commission .</td>\n",
+       "      <td>\\$ARG1 , executive director of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .</td>\n",
+       "      <td>\\$ARG1 , executive director of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .</td>\n",
+       "      <td>\\$ARG1 was born in \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>per:date_of_birth</td>\n",
+       "      <td>per:children</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .</td>\n",
+       "      <td>\\$ARG1 , executive director of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>`` At the moment the water issue does n't get enough attention in the climate negotiations , '' Anders Berntell , head of the Stockholm International Water Institute , told The Associated Press .</td>\n",
+       "      <td>\\$ARG1 , head of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>BEIJING , Dec 23 ( Xinhua ) Net profit of the central SOEs totaled 80226 billion yuan ( 12073 billion US dollars ) in the January-to-November period , said Wang Yong , head of the State - owned Assets Supervision and Administration Commission ( SASAC ) .</td>\n",
+       "      <td>\\$ARG1 , head of the \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .</td>\n",
+       "      <td>\\$ARG1 was born in \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>per:date_of_birth</td>\n",
+       "      <td>per:date_of_birth</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>`` It is difficult to put one country and its citizens in the same boat ... some people from Iran have very legitimate interests in Europe and are very trustworthy , '' said Pierre Mirabaud , head of the Swiss Bankers Association .</td>\n",
+       "      <td>\\$ARG1 , head of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>`` The conference is like the science Olympics , '' said Peter Agre , head of the American Association for the Advancement of Sciece ( AAAS ) , which is hosting the meeting in California as the Winter Olympics were in full swing in Vancouver , Canada .</td>\n",
+       "      <td>\\$ARG1 , head of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Net profit of the central SOEs totaled 802.26 billion yuan ( 120.73 billion U.S. dollars ) in the January-to-November period , said Wang Yong , head of the State - owned Assets Supervision and Administration Commission ( SASAC ) .</td>\n",
+       "      <td>\\$ARG1 , head of the \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>There is the danger , over time , that the United States could lose its pre-eminence in science and efforts to interest more young people in research are under way , said Alan I Leshner , head of the American Association for the Advancement of Science .</td>\n",
+       "      <td>\\$ARG1 , head of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Although military spending was n't usually a major part of the economic stimulus packages , it was n't cut either , '' said Sam Perlo-Freeman , head of the Military Expenditure Project at SIPRI .</td>\n",
+       "      <td>\\$ARG1 , head of \\$ARG2</td>\n",
+       "      <td>(A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                                                            sample  \\\n",
+       "0   His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .                                                  \n",
+       "0   `` It 's an issue for everybody in the state because peanuts are a big part of our economy , '' said Don Koehler , executive director of the Georgia Peanut Commission .                                                                                         \n",
+       "0   Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .                                         \n",
+       "0   Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .                                                                                                                                                  \n",
+       "0   Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .                                         \n",
+       "0   `` At the moment the water issue does n't get enough attention in the climate negotiations , '' Anders Berntell , head of the Stockholm International Water Institute , told The Associated Press .                                                              \n",
+       "0   BEIJING , Dec 23 ( Xinhua ) Net profit of the central SOEs totaled 80226 billion yuan ( 12073 billion US dollars ) in the January-to-November period , said Wang Yong , head of the State - owned Assets Supervision and Administration Commission ( SASAC ) .   \n",
+       "0   Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .                                                                                                                                                  \n",
+       "0   `` It is difficult to put one country and its citizens in the same boat ... some people from Iran have very legitimate interests in Europe and are very trustworthy , '' said Pierre Mirabaud , head of the Swiss Bankers Association .                          \n",
+       "0   `` The conference is like the science Olympics , '' said Peter Agre , head of the American Association for the Advancement of Sciece ( AAAS ) , which is hosting the meeting in California as the Winter Olympics were in full swing in Vancouver , Canada .     \n",
+       "0   Net profit of the central SOEs totaled 802.26 billion yuan ( 120.73 billion U.S. dollars ) in the January-to-November period , said Wang Yong , head of the State - owned Assets Supervision and Administration Commission ( SASAC ) .                           \n",
+       "0   There is the danger , over time , that the United States could lose its pre-eminence in science and efforts to interest more young people in research are under way , said Alan I Leshner , head of the American Association for the Advancement of Science .    \n",
+       "0   Although military spending was n't usually a major part of the economic stimulus packages , it was n't cut either , '' said Sam Perlo-Freeman , head of the Military Expenditure Project at SIPRI .                                                              \n",
+       "\n",
+       "                             extr_sample  \\\n",
+       "0  \\$ARG1 , who heads \\$ARG2               \n",
+       "0  \\$ARG1 , executive director of \\$ARG2   \n",
+       "0  \\$ARG1 , executive director of \\$ARG2   \n",
+       "0  \\$ARG1 was born in \\$ARG2               \n",
+       "0  \\$ARG1 , executive director of \\$ARG2   \n",
+       "0  \\$ARG1 , head of \\$ARG2                 \n",
+       "0  \\$ARG1 , head of the \\$ARG2             \n",
+       "0  \\$ARG1 was born in \\$ARG2               \n",
+       "0  \\$ARG1 , head of \\$ARG2                 \n",
+       "0  \\$ARG1 , head of \\$ARG2                 \n",
+       "0  \\$ARG1 , head of the \\$ARG2             \n",
+       "0  \\$ARG1 , head of \\$ARG2                 \n",
+       "0  \\$ARG1 , head of \\$ARG2                 \n",
+       "\n",
+       "                                                                                        pattern  \\\n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ who\\ heads\\ (A )?(a )?(The )?(the )?\\\\$ARG2                \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2   \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2   \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2                \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ executive\\ director\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2   \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2                  \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2                  \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ was\\ born\\ in\\ (A )?(a )?(The )?(the )?\\\\$ARG2                \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2                  \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2                  \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2                  \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2                  \n",
+       "0  (A )?(a )?(The )?(the )?\\\\$ARG1\\ ,\\ head\\ of\\ (A )?(a )?(The )?(the )?\\\\$ARG2                  \n",
+       "\n",
+       "                  weak_label                 gold_label  \n",
+       "0  org:top_members/employees  per:religion               \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  org:top_members/employees  org:alternate_names        \n",
+       "0  per:date_of_birth          per:children               \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  per:date_of_birth          per:date_of_birth          \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  org:top_members/employees  org:alternate_names        \n",
+       "0  org:top_members/employees  org:top_members/employees  \n",
+       "0  org:top_members/employees  org:top_members/employees  "
+      ]
+     },
+     "execution_count": 104,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_matches_filter_types.apply(lambda x: escape_dollar(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 105,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "65"
+      ]
+     },
+     "execution_count": 105,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_matches_filter_types.size"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "## Input matrices creation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "So, now we have a smaller dataset (only 60 found samples compared to 120 without argument types filtering), but much more clean. By the way: we have already done our first weakly data preprocessing :)\n",
+    "\n",
+    "Some weak labels are still not the same as the gold labels, but mostly this happens because they are hold in different parts of a sentence. For example, \"Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer.\" sentence definitely contains both relations: \"per:date_of_birth\" and \"per:children\" - so, our distant label is still true. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The final step would be a creation of the input matrices needed for knodle framework, namely:\n",
+    "\n",
+    "- X: input_features (sentence x features): containing information about the data samples\n",
+    "- Z: rule_matches (sentence x labelling): in our case labelling functions are patterns and this binary matrix contains information about which pattern match for each sentence\n",
+    "- T: mapping_rules_labels (labelling functions x classes): which pattern corresponds to which class\n",
+    "\n",
+    "We are not going to discuss here encoding of the data samples and creation of X (input_features) matrix, but let's construct Z (rule_matches) and T (mapping_rules_labels) matrices."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def z_matrix_creation(all_matches_filter_types, relation_patterns_df):\n",
+    "    \n",
+    "    # create empty matrix with columns as patterns + column \"sample\"\n",
+    "    sentences_lf_df_columns = [item for relation in relation_patterns_df[\"raw pattern\"] for item in relation] \n",
+    "    rule_matches = pd.DataFrame(columns=escape_dollar(sentences_lf_df_columns))\n",
+    "    rule_matches.insert(0, \"sample\", all_matches_filter_types[\"sample\"])\n",
+    "\n",
+    "    for index, row in all_matches_filter_types.iterrows():   # iterate over rows of found samples\n",
+    "        curr_sample, curr_pattern, curr_relation = row[0], row[2], row[3]\n",
+    "        pattern_index = list(relation_patterns_df.loc[curr_relation][\"regex pattern\"]).index(curr_pattern)    # find idx of current regex pattern in patterns lists in relation_patterns_df\n",
+    "        raw_pattern = relation_patterns_df.loc[curr_relation, \"raw pattern\"][pattern_index]   # take the corresponding raw pattern\n",
+    "        rule_matches.loc[(rule_matches[\"sample\"] == curr_sample), escape_dollar([raw_pattern])] = 1   # add 1 on the intersection of sample and pattern in Z matrix\n",
+    "\n",
+    "    return rule_matches.fillna(0)\n",
+    "\n",
+    "rule_matches = z_matrix_creation(all_matches_filter_types, relation_patterns_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sample</th>\n",
+       "      <th>\\$ARG1 ( \\$ARG2</th>\n",
+       "      <th>\\$ARG1 formerly known as \\$ARG2</th>\n",
+       "      <th>\\$ARG1 aka \\$ARG2</th>\n",
+       "      <th>\\$ARG1 ( also known as \\$ARG2 )</th>\n",
+       "      <th>\\$ARG1 ( born \\$ARG2 )</th>\n",
+       "      <th>\\$ARG1 ( born \\$ARG2 in</th>\n",
+       "      <th>\\$ARG1 ( \\$ARG2 -</th>\n",
+       "      <th>\\$ARG1 was born in \\$ARG2</th>\n",
+       "      <th>\\$ARG1 , executive director of \\$ARG2</th>\n",
+       "      <th>\\$ARG1 , head of \\$ARG2</th>\n",
+       "      <th>\\$ARG1 , who heads \\$ARG2</th>\n",
+       "      <th>\\$ARG1 , chief executive of \\$ARG2</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>`` It 's an issue for everybody in the state because peanuts are a big part of our economy , '' said Don Koehler , executive director of the Georgia Peanut Commission .</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                                                                      sample  \\\n",
+       "0   His main Shiite rival , Abdul Aziz al-Hakim , who heads the Islamic Supreme Council of Iraq , an influential Shiite political party that is part of Maliki 's ruling coalition , has also denounced the plans .            \n",
+       "0   `` It 's an issue for everybody in the state because peanuts are a big part of our economy , '' said Don Koehler , executive director of the Georgia Peanut Commission .                                                   \n",
+       "0   Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .   \n",
+       "0   Gwathmey was born in 1938 , the only child of painter Robert Gwathmey and his wife , Rosalie , a photographer .                                                                                                            \n",
+       "0   Water and its links to development , peace and conflict were key words in the annual sessions , Anders Berntell , executive director of Stockholm International Water Institute ( SIWI ) , said in his opening address .   \n",
+       "\n",
+       "   \\$ARG1 ( \\$ARG2   \\$ARG1 formerly known as \\$ARG2  \\$ARG1 aka \\$ARG2  \\\n",
+       "0  0                 0                                0                   \n",
+       "0  0                 0                                0                   \n",
+       "0  0                 0                                0                   \n",
+       "0  0                 0                                0                   \n",
+       "0  0                 0                                0                   \n",
+       "\n",
+       "   \\$ARG1 ( also known as \\$ARG2 )  \\$ARG1 ( born \\$ARG2 )  \\\n",
+       "0  0                                0                        \n",
+       "0  0                                0                        \n",
+       "0  0                                0                        \n",
+       "0  0                                0                        \n",
+       "0  0                                0                        \n",
+       "\n",
+       "   \\$ARG1 ( born \\$ARG2 in  \\$ARG1 ( \\$ARG2 -  \\$ARG1 was born in \\$ARG2  \\\n",
+       "0  0                        0                  0                           \n",
+       "0  0                        0                  0                           \n",
+       "0  0                        0                  0                           \n",
+       "0  0                        0                  1                           \n",
+       "0  0                        0                  0                           \n",
+       "\n",
+       "   \\$ARG1 , executive director of \\$ARG2  \\$ARG1 , head of \\$ARG2  \\\n",
+       "0  0                                      0                         \n",
+       "0  1                                      0                         \n",
+       "0  1                                      0                         \n",
+       "0  0                                      0                         \n",
+       "0  1                                      0                         \n",
+       "\n",
+       "   \\$ARG1 , who heads \\$ARG2  \\$ARG1 , chief executive of \\$ARG2  \n",
+       "0  1                          0                                   \n",
+       "0  0                          0                                   \n",
+       "0  0                          0                                   \n",
+       "0  0                          0                                   \n",
+       "0  0                          0                                   "
+      ]
+     },
+     "execution_count": 63,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rule_matches.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 76,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "mapping_rules_labels = pd.concat([pd.Series(escape_dollar(v), name=k).astype(str) for k, v in relation_patterns.items()], axis=1)\n",
+    "mapping_rules_labels = pd.get_dummies(mapping_rules_labels.stack()).sum(level=1).clip(upper=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>\\$ARG1 ( \\$ARG2</th>\n",
+       "      <th>\\$ARG1 ( \\$ARG2 -</th>\n",
+       "      <th>\\$ARG1 ( also known as \\$ARG2 )</th>\n",
+       "      <th>\\$ARG1 ( born \\$ARG2 )</th>\n",
+       "      <th>\\$ARG1 ( born \\$ARG2 in</th>\n",
+       "      <th>\\$ARG1 , chief executive of \\$ARG2</th>\n",
+       "      <th>\\$ARG1 , executive director of \\$ARG2</th>\n",
+       "      <th>\\$ARG1 , head of \\$ARG2</th>\n",
+       "      <th>\\$ARG1 , who heads \\$ARG2</th>\n",
+       "      <th>\\$ARG1 aka \\$ARG2</th>\n",
+       "      <th>\\$ARG1 formerly known as \\$ARG2</th>\n",
+       "      <th>\\$ARG1 was born in \\$ARG2</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>org:alternate_names</th>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>per:date_of_birth</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>org:top_members/employees</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                           \\$ARG1 ( \\$ARG2   \\$ARG1 ( \\$ARG2 -  \\\n",
+       "org:alternate_names        1                 0                   \n",
+       "per:date_of_birth          0                 1                   \n",
+       "org:top_members/employees  0                 0                   \n",
+       "\n",
+       "                           \\$ARG1 ( also known as \\$ARG2 )  \\\n",
+       "org:alternate_names        1                                 \n",
+       "per:date_of_birth          0                                 \n",
+       "org:top_members/employees  0                                 \n",
+       "\n",
+       "                           \\$ARG1 ( born \\$ARG2 )  \\$ARG1 ( born \\$ARG2 in  \\\n",
+       "org:alternate_names        0                       0                         \n",
+       "per:date_of_birth          1                       1                         \n",
+       "org:top_members/employees  0                       0                         \n",
+       "\n",
+       "                           \\$ARG1 , chief executive of \\$ARG2  \\\n",
+       "org:alternate_names        0                                    \n",
+       "per:date_of_birth          0                                    \n",
+       "org:top_members/employees  1                                    \n",
+       "\n",
+       "                           \\$ARG1 , executive director of \\$ARG2  \\\n",
+       "org:alternate_names        0                                       \n",
+       "per:date_of_birth          0                                       \n",
+       "org:top_members/employees  1                                       \n",
+       "\n",
+       "                           \\$ARG1 , head of \\$ARG2  \\$ARG1 , who heads \\$ARG2  \\\n",
+       "org:alternate_names        0                        0                           \n",
+       "per:date_of_birth          0                        0                           \n",
+       "org:top_members/employees  1                        1                           \n",
+       "\n",
+       "                           \\$ARG1 aka \\$ARG2  \\$ARG1 formerly known as \\$ARG2  \\\n",
+       "org:alternate_names        1                  1                                 \n",
+       "per:date_of_birth          0                  0                                 \n",
+       "org:top_members/employees  0                  0                                 \n",
+       "\n",
+       "                           \\$ARG1 was born in \\$ARG2  \n",
+       "org:alternate_names        0                          \n",
+       "per:date_of_birth          1                          \n",
+       "org:top_members/employees  0                          "
+      ]
+     },
+     "execution_count": 77,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mapping_rules_labels"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "## Finish"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we have not only a weakly supervised dataset, but also an appropriate input for the knodle framework. What is still to be done is the choosing of text features, converting them to matrix as well and pass all three matrices in knodle to make denoising and model training. Good luck! :) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ds_data_collect",
+   "language": "python",
+   "name": "ds_data_collect"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
\ No newline at end of file
diff --git a/examples/data_preprocessing/tac_based_dataset/[WIP]_patterns_aka_lfs/tac_based_re_dataset_preprocessor.py b/examples/data_preprocessing/tac_based_dataset/[WIP]_patterns_aka_lfs/tac_based_re_dataset_preprocessor.py
new file mode 100644
index 00000000..a6c64567
--- /dev/null
+++ b/examples/data_preprocessing/tac_based_dataset/[WIP]_patterns_aka_lfs/tac_based_re_dataset_preprocessor.py
@@ -0,0 +1,139 @@
+import argparse
+import logging
+import os
+import re
+import sys
+from typing import Union
+from pathlib import Path
+from joblib import dump
+
+import numpy as np
+import pandas as pd
+
+from tutorials.data_preprocessing.tac_based_dataset.utils.utils import (
+    get_analysed_conll_data, get_id, update_dict, get_match_matrix_row, save_dict, convert_pattern_to_regex
+)
+
+Z_MATRIX_OUTPUT = "z_matrix"
+T_MATRIX_OUTPUT = "t_matrix"
+TRAIN_SAMPLES_OUTPUT = "train_samples.csv"
+NO_PATTERN_TRAIN_SAMPLES_OUTPUT = "neg_train_samples.csv"
+DEV_SAMPLES_OUTPUT = "dev_samples.csv"
+
+logger = logging.getLogger(__name__)
+relation2id, pattern2id, pattern2regex, relation2patterns = {}, {}, {}, {}
+
+
+def collect_data(path_train_data: str, path_dev_data: str, path_patterns: str, path_labels: str, path_output: str) -> None:
+    """ This function reads train and dev data and saved resulted files to output directory"""
+    Path(path_output).mkdir(parents=True, exist_ok=True)
+
+    labels = _get_labels(path_labels)
+
+    _get_train_data(path_train_data, path_patterns, path_output, labels)
+    _get_dev_data(path_dev_data, path_output)
+
+    save_dict(relation2id, os.path.join(path_output, "relation2id.json"))
+    save_dict(pattern2id, os.path.join(path_output, "pattern2id.json"))
+
+
+def _get_labels(path_labels) -> list:
+    """ This function reads the labels from the file"""
+    with open(path_labels, encoding="UTF-8") as file:
+        return [re.sub("\n", "", line) for line in file.readlines()]
+
+
+def _get_train_data(path_train_data: str, path_patterns: str, path_output: str, labels: list) -> None:
+    """
+    This function processes the train data and save t_matrix, z_matrix and training set info in two DataFrames:
+    - DataFrame with samples where some pattern matched (samples as text, matched patterns, encoded gold labels)
+    - DataFrame with samples where no pattern matched (samples as text, encoded gold labels)
+    to output directory.
+    """
+    logger.info("Processing of train data has started")
+
+    rule_assignments_t = _get_t_matrix(path_patterns, labels)
+    train_samples, neg_train_samples = get_analysed_conll_data(path_train_data, pattern2regex, relation2id,
+                                                               perform_search=True)
+    rule_matches_z = np.array(list(train_samples["retrieved_patterns"]), dtype=np.int)
+    _save_train_data(rule_assignments_t, rule_matches_z, train_samples, neg_train_samples, path_output)
+
+    logger.info("Processing of train data has finished")
+
+
+def _get_t_matrix(path_patterns: str, labels: list) -> np.ndarray:
+    """ Create a T matrix of pattern-relation correspondences"""
+    t_matrix = []
+    with open(path_patterns, encoding="UTF-8") as inp:
+        for line in inp.readlines():
+            pattern = _read_pattern(line, labels)
+            if pattern:
+                t_matrix.append(pattern)
+    return np.asarray(list(filter(None, t_matrix)))
+
+
+def _read_pattern(pattern_line: str, labels: list) -> Union[None, list]:
+    """
+    Processing of pattern file line. If there is a pattern line, encode pattern and relation it corresponds to
+    with ids, turn pattern into regex. Save info to corresponding dicts and return information of pattern-relation
+    corresponding.
+    :return: a row of future T matrix as a list
+    """
+    if pattern_line.startswith("#") or pattern_line == "\n":  # take only meaningful strings
+        return None
+    relation, pattern = pattern_line.replace("\n", "").split(" ", 1)
+    if pattern in pattern2id:
+        return None
+    if relation not in labels:
+        logger.error("Relation {} is not in TACRED relations list. The pattern will be skipped".format(relation))
+        return None
+    relation_id = get_id(relation, relation2id)
+    pattern_id = get_id(pattern, pattern2id)
+    update_dict(relation_id, pattern_id, relation2patterns)
+    pattern2regex[pattern_id] = convert_pattern_to_regex(pattern)
+    return get_match_matrix_row(len(labels), [relation_id])
+
+
+def _save_train_data(
+        rule_assignments_t: np.ndarray, rule_matches_z: np.ndarray, train_samples: pd.DataFrame,
+        neg_train_samples: pd.DataFrame, path_output: str
+) -> None:
+    """
+    This function saves the training data to output directory
+    :param rule_assignments_t: t matrix
+    :param rule_matches_z: z matrix
+    :param train_samples: DataFrame with samples where some pattern matched (raw samples, matched patterns,
+    encoded gold labels)
+    :param neg_train_samples: DataFrame with samples with no pattern matched (raw samples, encoded gold labels)
+    """
+    dump(rule_assignments_t, os.path.join(path_output, T_MATRIX_OUTPUT))
+    dump(rule_matches_z, os.path.join(path_output, Z_MATRIX_OUTPUT))
+    train_samples.to_csv(os.path.join(path_output, TRAIN_SAMPLES_OUTPUT), columns=["sample", "raw_retrieved_patterns",
+                                                                                   "labels", "enc_labels"])
+    neg_train_samples.to_csv(os.path.join(path_output, NO_PATTERN_TRAIN_SAMPLES_OUTPUT),
+                             columns=["sample", "labels", "enc_labels"])
+
+
+def _get_dev_data(path_dev_data: str, path_output: str) -> None:
+    """
+    This function processes the development data and save it as DataFrame with samples as row text and gold labels
+    (encoded with ids) to output directory.
+    """
+    logger.info("Processing of dev data has started")
+
+    dev_data, _ = get_analysed_conll_data(path_dev_data, pattern2regex, relation2id)
+    dev_data.to_csv(os.path.join(path_output, DEV_SAMPLES_OUTPUT), columns=["sample", "enc_labels", "labels"])
+
+    logger.info("Processing of dev data has finished")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]))
+    parser.add_argument("--train_data", help="")
+    parser.add_argument("--dev_data", help="")
+    parser.add_argument("--patterns", help="")
+    parser.add_argument("--labels", help="List of labels")
+    parser.add_argument("--path_to_output", help="")
+
+    args = parser.parse_args()
+    collect_data(args.train_data, args.dev_data, args.patterns, args.labels, args.path_to_output)
diff --git a/examples/data_preprocessing/tac_based_dataset/__init__.py b/examples/data_preprocessing/tac_based_dataset/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/data_preprocessing/tac_based_dataset/entity_pairs_aka_lfs/Prepare_TAC_based_RE_dataset.ipynb b/examples/data_preprocessing/tac_based_dataset/entity_pairs_aka_lfs/Prepare_TAC_based_RE_dataset.ipynb
new file mode 100644
index 00000000..4e38d2eb
--- /dev/null
+++ b/examples/data_preprocessing/tac_based_dataset/entity_pairs_aka_lfs/Prepare_TAC_based_RE_dataset.ipynb
@@ -0,0 +1,1615 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Preprocessing of TAC-based Relation Extraction dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This notebook shows how to preprocess data in CONLL format, which is quite popular for storing the NLP datasets, for Knodle framework.\n",
+    "\n",
+    "To show how it works, we have taken a relation extraction dataset based on TAC KBP corpora (Surdeanu (2013)), also used in Roth (2014). The TAC dataset was annotated with entity pairs extracted from Freebase (Google (2014)) where corresponding relations have been mapped to the 41 TAC relations types as used in the TAC KBP challenges (e.g., per:schools_attended and org:members). \n",
+    "\n",
+    "In order to show the whole process of weak annotation, we have reconstructed the entity pairs and used them to annotate the dataset from scrath. As development and test sets we used the gold corpus annotated via crowdsourcing and human labeling from KBP (Zhang et al. (2017)).  \n",
+    "\n",
+    "Importantly, in this dataset we preserve the samples, where no rule matched, as __negative samples__, what is considered to be a good practice in many NLP tasks, e.g. relation extraction. \n",
+    "\n",
+    "The steps are the following:\n",
+    "- the input data files are downloaded from MINIO database: \n",
+    "    - raw train data saved in .conll format\n",
+    "    - gold-annotated dev data saved in .conll format\n",
+    "    - gold-annotated test data saved in .conll format\n",
+    "    - list of rules (namely, Freebase entity pairs) with corresponding classes\n",
+    "    - list of classes\n",
+    "- list of rules with corresponding classes is transformed to mapping_rules_labels t matrics\n",
+    "- the non-labelled train data are read from .conll file and annotated with entity pairs. Basing on them, rule_matches_z matrix and a DataFrame with train samples are generated\n",
+    "- the already annotated dev and test data are read from .conll file together with gold labels and stored as a DataFrame."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Imports"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Firstly, let's make some basic imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import argparse\n",
+    "import sys\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import logging\n",
+    "from typing import Dict, Union, Tuple\n",
+    "from minio import Minio\n",
+    "import random\n",
+    "from IPython.display import HTML\n",
+    "import csv\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import scipy.sparse as sp\n",
+    "from joblib import dump\n",
+    "from tqdm.auto import tqdm\n",
+    "\n",
+    "from knodle.trainer.utils import log_section\n",
+    "\n",
+    "pd.set_option('display.max_colwidth', -1)\n",
+    "np.set_printoptions(threshold=sys.maxsize)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'../../../data_from_minio/TAC'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# define the files names\n",
+    "Z_MATRIX_OUTPUT_TRAIN = \"train_rule_matches_z.lib\"\n",
+    "Z_MATRIX_OUTPUT_DEV = \"dev_rule_matches_z.lib\"\n",
+    "Z_MATRIX_OUTPUT_TEST = \"test_rule_matches_z.lib\"\n",
+    "\n",
+    "T_MATRIX_OUTPUT_TRAIN = \"mapping_rules_labels_t.lib\"\n",
+    "\n",
+    "TRAIN_SAMPLES_OUTPUT = \"df_train.lib\"\n",
+    "DEV_SAMPLES_OUTPUT = \"df_dev.lib\"\n",
+    "TEST_SAMPLES_OUTPUT = \"df_test.lib\"\n",
+    "\n",
+    "# file names for .csv files\n",
+    "TRAIN_SAMPLES_OUTPUT_CSV = \"df_train.csv\"\n",
+    "DEV_SAMPLES_OUTPUT_CSV = \"df_dev.csv\"\n",
+    "TEST_SAMPLES_OUTPUT_CSV = \"df_test.csv\"\n",
+    "\n",
+    "# define the path to the folder where the data will be stored\n",
+    "data_path = \"../../../data_from_minio/TAC\"\n",
+    "os.path.join(data_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Download the dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This dataset, as all datasets provided in Knodle, could be easily downloaded from Minio database with Minio client. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = Minio(\"knodle.dm.univie.ac.at\", secure=False)\n",
+    "files = [\"train.conll\", \"dev.conll\", \"test.conll\", \"labels.txt\", \"rules.csv\"]\n",
+    "\n",
+    "for file in tqdm(files):\n",
+    "    client.fget_object(\n",
+    "        bucket_name=\"knodle\",\n",
+    "        object_name=os.path.join(\"datasets/tac\", file),\n",
+    "        file_path=os.path.join(data_path, file),\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# set paths to input data\n",
+    "path_labels = os.path.join(data_path, \"labels.txt\")\n",
+    "path_rules = os.path.join(data_path, \"rules.csv\")\n",
+    "path_train_data = os.path.join(data_path, \"train.conll\")\n",
+    "path_dev_data = os.path.join(data_path, \"dev.conll\")\n",
+    "path_test_data = os.path.join(data_path, \"test.conll\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Labels & Rules Data Preprocessing¶"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Get labels"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "First, let's read labels from the file with the corresponding label ids."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "labels2ids = {}\n",
+    "with open(path_labels, encoding=\"UTF-8\") as file:\n",
+    "    for line in file.readlines():\n",
+    "        relation, relation_enc = line.replace(\"\\n\", \"\").split(\",\")\n",
+    "        labels2ids[relation] = int(relation_enc)\n",
+    "\n",
+    "num_classes = len(labels2ids)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'per:alternate_names': 0, 'per:date_of_birth': 1, 'per:age': 2, 'per:country_of_birth': 3, 'per:stateorprovince_of_birth': 4, 'per:city_of_birth': 5, 'per:origin': 6, 'per:date_of_death': 7, 'per:country_of_death': 8, 'per:stateorprovince_of_death': 9, 'per:city_of_death': 10, 'per:cause_of_death': 11, 'per:countries_of_residence': 12, 'per:stateorprovinces_of_residence': 13, 'per:cities_of_residence': 14, 'per:schools_attended': 15, 'per:title': 16, 'per:employee_of': 17, 'per:religion': 18, 'per:spouse': 19, 'per:children': 20, 'per:parents': 21, 'per:siblings': 22, 'per:other_family': 23, 'per:charges': 24, 'org:alternate_names': 25, 'org:members': 26, 'org:member_of': 27, 'org:subsidiaries': 28, 'org:political/religious_affiliation': 29, 'org:top_members/employees': 30, 'org:number_of_employees/members': 31, 'org:parents': 32, 'org:founded_by': 33, 'org:founded': 34, 'org:country_of_headquarters': 35, 'org:stateorprovince_of_headquarters': 36, 'org:city_of_headquarters': 37, 'org:shareholders': 38, 'org:website': 39, 'org:dissolved': 40}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(labels2ids)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Get rules"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Secondly, rules (in our case, entity pairs extracted from Freebase) that are stored in the separate csv file with corresponding _label_ and _label_id_ (_label_ to _label_id_ correspondance is the same as in file with labels list) are read and stored."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>rule</th>\n",
+       "      <th>rule_id</th>\n",
+       "      <th>label</th>\n",
+       "      <th>label_id</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ATG Art_Technology_Group</td>\n",
+       "      <td>0</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "      <td>25</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Union_Cycliste_Internationale UCI</td>\n",
+       "      <td>1</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "      <td>25</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>UCI Union_Cycliste_Internationale</td>\n",
+       "      <td>2</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "      <td>25</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Hanwha 한화</td>\n",
+       "      <td>3</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "      <td>25</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Radio_Free_Europe Radio_Liberty</td>\n",
+       "      <td>4</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "      <td>25</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>247315</th>\n",
+       "      <td>Ginger_Baker drums</td>\n",
+       "      <td>212032</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>16</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>247316</th>\n",
+       "      <td>painter Qi_Baishi</td>\n",
+       "      <td>212033</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>16</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>247317</th>\n",
+       "      <td>Qi_Baishi painter</td>\n",
+       "      <td>212034</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>16</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>247318</th>\n",
+       "      <td>engineer David_Lennox</td>\n",
+       "      <td>212035</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>16</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>247319</th>\n",
+       "      <td>David_Lepper politician</td>\n",
+       "      <td>212036</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>16</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>247320 rows × 4 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                     rule  rule_id                label  \\\n",
+       "0       ATG Art_Technology_Group           0        org:alternate_names   \n",
+       "1       Union_Cycliste_Internationale UCI  1        org:alternate_names   \n",
+       "2       UCI Union_Cycliste_Internationale  2        org:alternate_names   \n",
+       "3       Hanwha 한화                          3        org:alternate_names   \n",
+       "4       Radio_Free_Europe Radio_Liberty    4        org:alternate_names   \n",
+       "...                                 ...   ..                        ...   \n",
+       "247315  Ginger_Baker drums                 212032   per:title             \n",
+       "247316  painter Qi_Baishi                  212033   per:title             \n",
+       "247317  Qi_Baishi painter                  212034   per:title             \n",
+       "247318  engineer David_Lennox              212035   per:title             \n",
+       "247319  David_Lepper politician            212036   per:title             \n",
+       "\n",
+       "        label_id  \n",
+       "0       25        \n",
+       "1       25        \n",
+       "2       25        \n",
+       "3       25        \n",
+       "4       25        \n",
+       "...     ..        \n",
+       "247315  16        \n",
+       "247316  16        \n",
+       "247317  16        \n",
+       "247318  16        \n",
+       "247319  16        \n",
+       "\n",
+       "[247320 rows x 4 columns]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rules = pd.read_csv(path_rules)\n",
+    "num_rules_from_file = len(rules)\n",
+    "rules"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Most rules and classes have one-to-one correspondance. However, there could be cases where a rule correponds to different classes. For example, _\"Oracle, New_York\"_ entity pair can reflect to both _org:stateorprovince_of_headquarters_ and _org:city_of_headquarters_ relations. In such cases information about all correponding classed will be saved and reflected in the _mapping_rules_labels_t_ matrix we are going to build in the next section."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Get rules to classes correspondance matrix"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Before that, basing on this dataframe let's build 2 dictionaries that we are going to use later:\n",
+    "- rule to rule ids correspondings\n",
+    "- rule ids to label ids correspondings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of rules: 212037\n"
+     ]
+    }
+   ],
+   "source": [
+    "rule2rule_id = dict(zip(rules[\"rule\"], rules[\"rule_id\"]))\n",
+    "\n",
+    "rules_n_label_ids = rules[[\"rule_id\", \"label_id\"]].groupby('rule_id')\n",
+    "rule2label = rules_n_label_ids['label_id'].apply(lambda s: s.tolist()).to_dict()\n",
+    "\n",
+    "num_rules = max(rules.rule_id.values) + 1\n",
+    "print(f\"Number of rules: {num_rules}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Finally, let's the build `mapping_rules_labels_t` matrix with the information about which rule corresponds to which class. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_mapping_rules_labels_t(rule2label: Dict, num_classes: int) -> np.ndarray:\n",
+    "    \"\"\" Function calculates t matrix (rules x labels) using the known correspondence of relations to decision rules \"\"\"\n",
+    "    mapping_rules_labels_t = np.zeros([len(rule2label), num_classes])\n",
+    "    for rule, labels in rule2label.items():\n",
+    "        mapping_rules_labels_t[rule, labels] = 1\n",
+    "    return mapping_rules_labels_t\n",
+    "\n",
+    "mapping_rules_labels_t = get_mapping_rules_labels_t(rule2label, num_classes)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Train data preprocessing"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Train data should be annotated with rules we already have. Remember, there is no gold labels (as opposite to evaluation and test data). To preserve samples without rule matches as negative samples in the training set, we do not eliminate them but add them to the preprocessed data with empty rule and rule_id value. \n",
+    "\n",
+    "So, the annotation is done in the following way: \n",
+    "- the sentences are extracted from .conll file\n",
+    "- a pair of tokens tagged as object and subject are looked up in rules list\n",
+    "- if they form any rule from the rules list, this sentence is added to the train set. The matched rule and rule id is added accordingly.\n",
+    "- if they are not, this sentence is added to the train set with empty rule match"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def count_file_lines(file_name: str) -> int:\n",
+    "    \"\"\" Count the number of line in a file \"\"\"\n",
+    "    with open(file_name) as f:\n",
+    "        return len(f.readlines())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "#\tindex\ttoken\tsubj\tsubj_type\tobj\tobj_type\tstanford_pos\tstanford_ner\tstanford_deprel\tstanford_head\n",
+      "\n",
+      "# id=E0065795:0-pos docid=E0065795:0 reln=org:alternate_names\n",
+      "\n",
+      "1\tProfile\t_\t_\t_\t_\tVB\tO\tadvmod\t16\n",
+      "\n",
+      "2\t,\t_\t_\t_\t_\t,\tO\tpunct\t16\n",
+      "\n",
+      "3\tbasic\t_\t_\t_\t_\tJJ\tO\tamod\t4\n",
+      "\n",
+      "4\tinformation\t_\t_\t_\t_\tNN\tO\tcompound\t5\n",
+      "\n",
+      "5\tATG\t_\t_\tOBJECT\tORG\tNNP\tORG\tnsubj\t16\n",
+      "\n",
+      "6\t(\t_\t_\t_\t_\t-LRB-\tO\tpunct\t11\n",
+      "\n",
+      "7\tArt\tSUBJECT\tORGANIZATION\t_\t_\tNNP\tORG\tcompound\t9\n",
+      "\n",
+      "8\tTechnology\tSUBJECT\tORGANIZATION\t_\t_\tNNP\tORG\tcompound\t9\n",
+      "\n",
+      "9\tGroup\tSUBJECT\tORGANIZATION\t_\t_\tNNP\tORG\tnmod\t11\n",
+      "\n",
+      "10\t,\t_\t_\t_\t_\t,\tO\tpunct\t11\n",
+      "\n",
+      "11\tInc.\t_\t_\t_\t_\tNNP\tGPE\tappos\t5\n",
+      "\n",
+      "12\t,\t_\t_\t_\t_\t,\tO\tpunct\t11\n",
+      "\n",
+      "13\tNASDAQ\t_\t_\t_\t_\tNNP\tORG\tnpadvmod\t11\n",
+      "\n",
+      "14\t:\t_\t_\t_\t_\t:\tO\tpunct\t13\n",
+      "\n",
+      "15\t)\t_\t_\t_\t_\t-RRB-\tO\tpunct\t11\n",
+      "\n",
+      "16\tmakes\t_\t_\t_\t_\tVBZ\tO\tROOT\t16\n",
+      "\n",
+      "17\tsoftware\t_\t_\t_\t_\tNN\tO\tdobj\t16\n",
+      "\n",
+      "18\tand\t_\t_\t_\t_\tCC\tO\tcc\t16\n",
+      "\n",
+      "19\tdelivers\t_\t_\t_\t_\tVBZ\tO\tconj\t16\n",
+      "\n",
+      "20\te\t_\t_\t_\t_\tNN\tO\tnmod\t22\n",
+      "\n",
+      "21\t-\t_\t_\t_\t_\tHYPH\tO\tpunct\t22\n",
+      "\n",
+      "22\tcommerce\t_\t_\t_\t_\tNN\tO\tnmod\t26\n",
+      "\n",
+      "23\tand\t_\t_\t_\t_\tCC\tO\tcc\t22\n",
+      "\n",
+      "24\tWeb\t_\t_\t_\t_\tNN\tO\tcompound\t25\n",
+      "\n",
+      "25\tmarketing\t_\t_\t_\t_\tNN\tO\tconj\t22\n",
+      "\n",
+      "26\tsolutions\t_\t_\t_\t_\tNNS\tO\tdobj\t19\n",
+      "\n",
+      "27\tthat\t_\t_\t_\t_\tIN\tO\tdobj\t31\n",
+      "\n",
+      "28\tmany\t_\t_\t_\t_\tJJ\tO\tamod\t30\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "train_data = open(path_train_data)\n",
+    "for i in range(30):\n",
+    "    line = train_data.readline()\n",
+    "    print(line)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_subj_obj_middle_words(line: str, subj: list, obj: list, subj_min_token_id: int, obj_min_token_id: int, sample: str):\n",
+    "    splitted_line = line.split(\"\\t\")\n",
+    "    token = splitted_line[1]\n",
+    "    if splitted_line[2] == \"SUBJECT\":\n",
+    "        if not subj_min_token_id:\n",
+    "            subj_min_token_id = int(splitted_line[0])\n",
+    "        subj.append(token)\n",
+    "        sample += \" \" + token\n",
+    "    elif splitted_line[4] == \"OBJECT\":\n",
+    "        if not obj_min_token_id:\n",
+    "            obj_min_token_id = int(splitted_line[0])\n",
+    "        obj.append(token)\n",
+    "        sample += \" \" + token\n",
+    "    else:\n",
+    "        if (bool(subj) and not bool(obj)) or (not bool(subj) and bool(obj)):\n",
+    "            sample += \" \" + token\n",
+    "    return subj, obj, subj_min_token_id, obj_min_token_id, sample\n",
+    "\n",
+    "def get_rule_n_rule_id(subj: list, obj: list, subj_min_token_id: int, obj_min_token_id: int, rule2rule_id: dict) -> Union[Tuple[str, int], Tuple[None, None]]:\n",
+    "    if subj_min_token_id < obj_min_token_id:\n",
+    "        rule = \"_\".join(subj) + \" \" + \"_\".join(obj)\n",
+    "    else:\n",
+    "        rule = \"_\".join(obj) + \" \" + \"_\".join(subj)\n",
+    "    if rule in rule2rule_id.keys():\n",
+    "        return rule, rule2rule_id[rule]\n",
+    "    return None, None\n",
+    "\n",
+    "def encode_labels(label: str, label2id: dict) -> Union[int, None]:\n",
+    "    \"\"\" Encodes labels with corresponding labels id. If relation is unknown, adds it to the dict with new label id \"\"\"\n",
+    "    if label in label2id:\n",
+    "        return label2id[label]\n",
+    "    else:\n",
+    "        print(f\"Warning! There is a {label} label found which is not in the list of TAC relations! The sentence will be skipped.\")\n",
+    "        return None\n",
+    "\n",
+    "def print_progress(processed_lines: int, num_lines: int) -> None:\n",
+    "    if processed_lines % (int(round(num_lines / 10))) == 0:\n",
+    "        print(f\"Processed {processed_lines / num_lines * 100 :0.0f}%\")\n",
+    "\n",
+    "\n",
+    "def annotate_conll_data_with_lfs(conll_data: str, rule2rule_id: Dict, labels2ids: Dict = None) -> pd.DataFrame:\n",
+    "    num_lines = count_file_lines(conll_data)\n",
+    "    processed_lines = 0\n",
+    "    samples, rules, enc_rules, labels, enc_labels = [], [], [], [], []\n",
+    "    with open(conll_data, encoding='utf-8') as f:\n",
+    "        for line in f:\n",
+    "            processed_lines += 1\n",
+    "            line = line.strip()\n",
+    "            if line.startswith(\"# id=\"):  # Instance starts\n",
+    "                sample = \"\"\n",
+    "                subj, obj = [], []\n",
+    "                subj_min_token_id, obj_min_token_id = None, None\n",
+    "                if labels2ids:\n",
+    "                    label = line.split(\" \")[3][5:]\n",
+    "                    label_id = encode_labels(label, labels2ids)\n",
+    "            elif line == \"\":  # Instance ends\n",
+    "                if len(subj) == 0 or len(obj) == 0:      # there is a mistake in sample annotation, and no token was annotated as subj/obj \n",
+    "                    continue\n",
+    "                rule, rule_id = get_rule_n_rule_id(subj, obj, subj_min_token_id, obj_min_token_id, rule2rule_id)\n",
+    "                samples.append(sample.lstrip())\n",
+    "                rules.append(rule)\n",
+    "                enc_rules.append(rule_id)\n",
+    "                if labels2ids and label_id is not None:\n",
+    "                    labels.append(label)\n",
+    "                    enc_labels.append(label_id)\n",
+    "            elif line.startswith(\"#\"):  # comment\n",
+    "                continue\n",
+    "            else:\n",
+    "                subj, obj, subj_min_token_id, obj_min_token_id, sample = extract_subj_obj_middle_words(line, subj, obj, subj_min_token_id, obj_min_token_id, sample)\n",
+    "            print_progress(processed_lines, num_lines)\n",
+    "            \n",
+    "    print(f\"Preprocessing of {conll_data.split('/')[-1]} file is finished.\")\n",
+    "    if labels2ids:\n",
+    "        return pd.DataFrame.from_dict({\"samples\": samples, \"rules\": rules, \"enc_rules\": enc_rules, \"labels\": labels, \"enc_labels\": enc_labels}) \n",
+    "    return pd.DataFrame.from_dict({\"samples\": samples, \"rules\": rules, \"enc_rules\": enc_rules})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed 10%\n",
+      "Processed 20%\n",
+      "Processed 30%\n",
+      "Processed 40%\n",
+      "Processed 50%\n",
+      "Processed 60%\n",
+      "Processed 70%\n",
+      "Processed 80%\n",
+      "Processed 90%\n",
+      "Preprocessing of train.conll file is finished.\n"
+     ]
+    }
+   ],
+   "source": [
+    "train_data = annotate_conll_data_with_lfs(path_train_data, rule2rule_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>samples</th>\n",
+       "      <th>rules</th>\n",
+       "      <th>enc_rules</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ATG ( Art Technology Group</td>\n",
+       "      <td>ATG Art_Technology_Group</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>) makes software and delivers e - commerce</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Union Cycliste Internationale ( UCI</td>\n",
+       "      <td>Union_Cycliste_Internationale UCI</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1987 by CTCA and has been awarded the distinguished level</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Union Cycliste Internationale ( UCI</td>\n",
+       "      <td>Union_Cycliste_Internationale UCI</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                     samples  \\\n",
+       "0  ATG ( Art Technology Group                                  \n",
+       "1  ) makes software and delivers e - commerce                  \n",
+       "2  Union Cycliste Internationale ( UCI                         \n",
+       "3  1987 by CTCA and has been awarded the distinguished level   \n",
+       "4  Union Cycliste Internationale ( UCI                         \n",
+       "\n",
+       "                               rules  enc_rules  \n",
+       "0  ATG Art_Technology_Group           0.0        \n",
+       "1  None                              NaN         \n",
+       "2  Union_Cycliste_Internationale UCI  1.0        \n",
+       "3  None                              NaN         \n",
+       "4  Union_Cycliste_Internationale UCI  1.0        "
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_data.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "After that we could build a rule_matches_z matrix for train data and save it as a sparse matrix ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_rule_matches_z_matrix (data: pd.DataFrame, num_rules: int) -> sp.csr_matrix:\n",
+    "    \"\"\"\n",
+    "    Function calculates the z matrix (samples x rules)\n",
+    "    data: pd.DataFrame (samples, matched rules, matched rules id )\n",
+    "    output: sparse z matrix\n",
+    "    \"\"\"\n",
+    "    data_without_nan = data.reset_index().dropna()\n",
+    "    rule_matches_z_matrix_sparse = sp.csr_matrix(\n",
+    "        (\n",
+    "            np.ones(len(data_without_nan['index'].values)),\n",
+    "            (data_without_nan['index'].values, data_without_nan['enc_rules'].values)\n",
+    "        ),\n",
+    "        shape=(len(data.index), num_rules)\n",
+    "    )\n",
+    "    return rule_matches_z_matrix_sparse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_rule_matches_z = get_rule_matches_z_matrix(train_data, num_rules)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Dev & Test data preprocessing¶"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The validation and test data are to be read from the corresponding input files. Although the gold label are known and  could be simply received from the same input conll data, we still anotate the dev and test data with the same rules we used to annotate the train data (namely, Freebase entity pairs). That is done in order to lately evaluate the rules and get a baseline result by comparing the known gold labels and the weakly labels. However, because of the rules specificity, there is a very small amount of matched rules in dev and test data. That is why in final DataFrame for most of the samples _rules_ and _enc_rules_ values equal None.\n",
+    "\n",
+    "Apart from the 41 \"meaningful\" relations, there are also samples which are annotated as _\"no_relation\"_ samples in validation and test data. That's why we need to add one more class to the _labels2ids_ dictionary. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "labels2ids[\"no_relation\"] = max(labels2ids.values()) + 1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we can process the development and test data. We shall use the same function as for processing of training data with one difference: the labels will be also read and stored for each sample. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed 10%\n",
+      "Processed 20%\n",
+      "Processed 30%\n",
+      "Processed 40%\n",
+      "Processed 50%\n",
+      "Processed 60%\n",
+      "Processed 70%\n",
+      "Processed 80%\n",
+      "Processed 90%\n",
+      "Preprocessing of dev.conll file is finished.\n",
+      "Processed 10%\n",
+      "Processed 20%\n",
+      "Processed 30%\n",
+      "Processed 40%\n",
+      "Processed 50%\n",
+      "Processed 60%\n",
+      "Processed 70%\n",
+      "Processed 80%\n",
+      "Processed 90%\n",
+      "Processed 100%\n",
+      "Preprocessing of test.conll file is finished.\n"
+     ]
+    }
+   ],
+   "source": [
+    "dev_data = annotate_conll_data_with_lfs(path_dev_data, rule2rule_id, labels2ids)\n",
+    "test_data = annotate_conll_data_with_lfs(path_test_data, rule2rule_id, labels2ids)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>samples</th>\n",
+       "      <th>rules</th>\n",
+       "      <th>enc_rules</th>\n",
+       "      <th>labels</th>\n",
+       "      <th>enc_labels</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Douglas Flint will become chairman</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>16</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Jeffrey White in mid-February issued an injunction against Wikileaks after the Zurich-based Bank Julius Baer</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>no_relation</td>\n",
+       "      <td>41</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>PARIS 2009-07-07 11:07:32 UTC French media earlier reported that Montcourt</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>per:city_of_death</td>\n",
+       "      <td>10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>current holdings of Blackstone-operated funds include Universal Orlando , Cadbury Schweppes , Freedom Communications</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>no_relation</td>\n",
+       "      <td>41</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Nepali government and the guerrillas reached in an understanding during summit talks held on July 16 at Prime Minister Girija Prashad Koirala</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>no_relation</td>\n",
+       "      <td>41</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                         samples  \\\n",
+       "0  Douglas Flint will become chairman                                                                                                              \n",
+       "1  Jeffrey White in mid-February issued an injunction against Wikileaks after the Zurich-based Bank Julius Baer                                    \n",
+       "2  PARIS 2009-07-07 11:07:32 UTC French media earlier reported that Montcourt                                                                      \n",
+       "3  current holdings of Blackstone-operated funds include Universal Orlando , Cadbury Schweppes , Freedom Communications                            \n",
+       "4  Nepali government and the guerrillas reached in an understanding during summit talks held on July 16 at Prime Minister Girija Prashad Koirala   \n",
+       "\n",
+       "  rules  enc_rules             labels  enc_labels  \n",
+       "0  None NaN         per:title          16          \n",
+       "1  None NaN         no_relation        41          \n",
+       "2  None NaN         per:city_of_death  10          \n",
+       "3  None NaN         no_relation        41          \n",
+       "4  None NaN         no_relation        41          "
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dev_data.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We also provide `rule_matches_z` matrices for dev and test data in order to calculate the simple majority baseline. They won't be used in any of the denoising algorithms provided in Knodle."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dev_rule_matches_z = get_rule_matches_z_matrix(dev_data, num_rules)\n",
+    "test_rule_matches_z = get_rule_matches_z_matrix(test_data, num_rules)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Statistics"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's collect some statistics of the data we collected."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of rules: 212037\n",
+      "Dimension of t matrix: (212037, 41)\n",
+      "Number of samples in train set: 1937211\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Number of rules: {num_rules}\")\n",
+    "print(f\"Dimension of t matrix: {mapping_rules_labels_t.shape}\")\n",
+    "print(f\"Number of samples in train set: {len(train_data)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of samples in dev set: 5368\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th>enc_labels</th>\n",
+       "      <th>labels</th>\n",
+       "      <th>count</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>41</td>\n",
+       "      <td>no_relation</td>\n",
+       "      <td>4015</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>217</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>139</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17</td>\n",
+       "      <td>per:employee_of</td>\n",
+       "      <td>101</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>25</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "      <td>87</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>per:age</td>\n",
+       "      <td>60</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>per:countries_of_residence</td>\n",
+       "      <td>59</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>per:date_of_death</td>\n",
+       "      <td>54</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>per:origin</td>\n",
+       "      <td>49</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14</td>\n",
+       "      <td>per:cities_of_residence</td>\n",
+       "      <td>46</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>per:cause_of_death</td>\n",
+       "      <td>43</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>35</td>\n",
+       "      <td>org:country_of_headquarters</td>\n",
+       "      <td>41</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19</td>\n",
+       "      <td>per:spouse</td>\n",
+       "      <td>38</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>28</td>\n",
+       "      <td>org:subsidiaries</td>\n",
+       "      <td>34</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>per:city_of_death</td>\n",
+       "      <td>28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>24</td>\n",
+       "      <td>per:charges</td>\n",
+       "      <td>24</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>37</td>\n",
+       "      <td>org:city_of_headquarters</td>\n",
+       "      <td>23</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>per:children</td>\n",
+       "      <td>22</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>39</td>\n",
+       "      <td>org:website</td>\n",
+       "      <td>22</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>32</td>\n",
+       "      <td>org:parents</td>\n",
+       "      <td>21</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>26</td>\n",
+       "      <td>org:members</td>\n",
+       "      <td>20</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>33</td>\n",
+       "      <td>org:founded_by</td>\n",
+       "      <td>19</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>36</td>\n",
+       "      <td>org:stateorprovince_of_headquarters</td>\n",
+       "      <td>19</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>34</td>\n",
+       "      <td>org:founded</td>\n",
+       "      <td>18</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>per:stateorprovince_of_death</td>\n",
+       "      <td>15</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>per:stateorprovinces_of_residence</td>\n",
+       "      <td>14</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18</td>\n",
+       "      <td>per:religion</td>\n",
+       "      <td>14</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>23</td>\n",
+       "      <td>per:other_family</td>\n",
+       "      <td>12</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>27</td>\n",
+       "      <td>org:member_of</td>\n",
+       "      <td>12</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>per:country_of_death</td>\n",
+       "      <td>11</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>38</td>\n",
+       "      <td>org:shareholders</td>\n",
+       "      <td>11</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>21</td>\n",
+       "      <td>per:parents</td>\n",
+       "      <td>11</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>0</td>\n",
+       "      <td>per:alternate_names</td>\n",
+       "      <td>11</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>per:date_of_birth</td>\n",
+       "      <td>10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15</td>\n",
+       "      <td>per:schools_attended</td>\n",
+       "      <td>9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>31</td>\n",
+       "      <td>org:number_of_employees/members</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>per:city_of_birth</td>\n",
+       "      <td>8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>22</td>\n",
+       "      <td>per:siblings</td>\n",
+       "      <td>6</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>org:dissolved</td>\n",
+       "      <td>5</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>per:country_of_birth</td>\n",
+       "      <td>5</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>29</td>\n",
+       "      <td>org:political/religious_affiliation</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>per:stateorprovince_of_birth</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(f\"Number of samples in dev set: {len(dev_data)}\")\n",
+    "dev_stat = dev_data.groupby(['enc_labels','labels'])['samples'].count().sort_values(ascending=False).reset_index(name='count')\n",
+    "HTML(dev_stat.to_html(index=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of samples in test set: 18660\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th>enc_labels</th>\n",
+       "      <th>labels</th>\n",
+       "      <th>count</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>41</td>\n",
+       "      <td>no_relation</td>\n",
+       "      <td>14517</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16</td>\n",
+       "      <td>per:title</td>\n",
+       "      <td>626</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>org:top_members/employees</td>\n",
+       "      <td>409</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17</td>\n",
+       "      <td>per:employee_of</td>\n",
+       "      <td>351</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>per:age</td>\n",
+       "      <td>269</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>25</td>\n",
+       "      <td>org:alternate_names</td>\n",
+       "      <td>242</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14</td>\n",
+       "      <td>per:cities_of_residence</td>\n",
+       "      <td>233</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>per:countries_of_residence</td>\n",
+       "      <td>199</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>per:origin</td>\n",
+       "      <td>164</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>24</td>\n",
+       "      <td>per:charges</td>\n",
+       "      <td>125</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>35</td>\n",
+       "      <td>org:country_of_headquarters</td>\n",
+       "      <td>123</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>21</td>\n",
+       "      <td>per:parents</td>\n",
+       "      <td>111</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>per:stateorprovinces_of_residence</td>\n",
+       "      <td>97</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>37</td>\n",
+       "      <td>org:city_of_headquarters</td>\n",
+       "      <td>94</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>33</td>\n",
+       "      <td>org:founded_by</td>\n",
+       "      <td>89</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>23</td>\n",
+       "      <td>per:other_family</td>\n",
+       "      <td>88</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>32</td>\n",
+       "      <td>org:parents</td>\n",
+       "      <td>76</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19</td>\n",
+       "      <td>per:spouse</td>\n",
+       "      <td>73</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>22</td>\n",
+       "      <td>per:siblings</td>\n",
+       "      <td>69</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>per:date_of_death</td>\n",
+       "      <td>62</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18</td>\n",
+       "      <td>per:religion</td>\n",
+       "      <td>60</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>36</td>\n",
+       "      <td>org:stateorprovince_of_headquarters</td>\n",
+       "      <td>58</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>per:cause_of_death</td>\n",
+       "      <td>58</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>28</td>\n",
+       "      <td>org:subsidiaries</td>\n",
+       "      <td>52</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>34</td>\n",
+       "      <td>org:founded</td>\n",
+       "      <td>47</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>per:children</td>\n",
+       "      <td>41</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>26</td>\n",
+       "      <td>org:members</td>\n",
+       "      <td>41</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15</td>\n",
+       "      <td>per:schools_attended</td>\n",
+       "      <td>37</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>27</td>\n",
+       "      <td>org:member_of</td>\n",
+       "      <td>36</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>per:city_of_death</td>\n",
+       "      <td>36</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>39</td>\n",
+       "      <td>org:website</td>\n",
+       "      <td>36</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>31</td>\n",
+       "      <td>org:number_of_employees/members</td>\n",
+       "      <td>22</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>29</td>\n",
+       "      <td>org:political/religious_affiliation</td>\n",
+       "      <td>17</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>per:country_of_death</td>\n",
+       "      <td>16</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>per:stateorprovince_of_death</td>\n",
+       "      <td>15</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>38</td>\n",
+       "      <td>org:shareholders</td>\n",
+       "      <td>15</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>0</td>\n",
+       "      <td>per:alternate_names</td>\n",
+       "      <td>13</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>per:stateorprovince_of_birth</td>\n",
+       "      <td>11</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>per:date_of_birth</td>\n",
+       "      <td>10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>per:city_of_birth</td>\n",
+       "      <td>9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>per:country_of_birth</td>\n",
+       "      <td>9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>org:dissolved</td>\n",
+       "      <td>4</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(f\"Number of samples in test set: {len(test_data)}\")\n",
+    "test_stat = test_data.groupby(['enc_labels','labels'])['samples'].count().sort_values(ascending=False).reset_index(name='count')\n",
+    "HTML(test_stat.to_html(index=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save files"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "... and we save all the data we got. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['../../../data_from_minio/TAC/processed/test_rule_matches_z.lib']"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "Path(os.path.join(data_path, \"processed\")).mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "dump(sp.csr_matrix(mapping_rules_labels_t), os.path.join(data_path, \"processed\", T_MATRIX_OUTPUT_TRAIN))\n",
+    "\n",
+    "dump(train_data[\"samples\"], os.path.join(data_path, \"processed\", TRAIN_SAMPLES_OUTPUT))\n",
+    "train_data[\"samples\"].to_csv(os.path.join(data_path, \"processed\", TRAIN_SAMPLES_OUTPUT_CSV), header=True)\n",
+    "dump(train_rule_matches_z, os.path.join(data_path, \"processed\", Z_MATRIX_OUTPUT_TRAIN))\n",
+    "\n",
+    "dump(dev_data[[\"samples\", \"labels\", \"enc_labels\"]], os.path.join(data_path, \"processed\", DEV_SAMPLES_OUTPUT))\n",
+    "dev_data[[\"samples\", \"labels\", \"enc_labels\"]].to_csv(os.path.join(data_path, \"processed\", DEV_SAMPLES_OUTPUT_CSV), header=True)\n",
+    "dump(dev_rule_matches_z, os.path.join(data_path, \"processed\", Z_MATRIX_OUTPUT_DEV))\n",
+    "\n",
+    "dump(test_data[[\"samples\", \"labels\", \"enc_labels\"]], os.path.join(data_path, \"processed\", TEST_SAMPLES_OUTPUT))\n",
+    "test_data[[\"samples\", \"labels\", \"enc_labels\"]].to_csv(os.path.join(data_path, \"processed\", TEST_SAMPLES_OUTPUT_CSV), header=True)\n",
+    "dump(test_rule_matches_z, os.path.join(data_path, \"processed\", Z_MATRIX_OUTPUT_TEST))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Finish"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Congrats! Now we have all the data we need to launch Knodle on weakly-annotated TAC-based data."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
\ No newline at end of file
diff --git a/examples/data_preprocessing/tac_based_dataset/entity_pairs_aka_lfs/requirements.txt b/examples/data_preprocessing/tac_based_dataset/entity_pairs_aka_lfs/requirements.txt
new file mode 100644
index 00000000..15a9317d
--- /dev/null
+++ b/examples/data_preprocessing/tac_based_dataset/entity_pairs_aka_lfs/requirements.txt
@@ -0,0 +1,15 @@
+numpy~=1.19.2
+scipy~=1.5.3
+joblib~=0.17.0
+pandas~=1.1.5
+pytest~=6.1.2
+torch~=1.7.1
+scikit-learn~=0.21.3
+minio~=7.0.1
+tqdm~=4.50.2
+transformers~=3.4.0
+matplotlib~=3.3.3
+snorkel~=0.9.6
+setuptools~=51.3.3
+python-dotenv~=0.15.0
+spacy~=2.3.2
\ No newline at end of file
diff --git a/examples/data_preprocessing/tac_based_dataset/utils/__init__.py b/examples/data_preprocessing/tac_based_dataset/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/trainer/[WIP]_baseline/baseline_training_example.py b/examples/trainer/[WIP]_baseline/baseline_training_example.py
new file mode 100644
index 00000000..e9f8c904
--- /dev/null
+++ b/examples/trainer/[WIP]_baseline/baseline_training_example.py
@@ -0,0 +1,77 @@
+import logging
+import os
+
+from torch import Tensor
+from torch.optim import SGD
+from torch.utils.data import TensorDataset
+
+from knodle.data.download import MinioConnector
+from knodle.model.logistic_regression_model import (
+    LogisticRegressionModel,
+)
+from knodle.trainer.baseline.baseline import NoDenoisingTrainer
+from knodle.trainer.baseline.majority_config import TrainerConfig
+from tutorials.ImdbDataset.utils import init_logger
+from tutorials.utils import create_tfidf_values, read_train_dev_test
+
+logger = logging.getLogger(__name__)
+
+OUTPUT_CLASSES = 2
+RANDOM_STATE = 123
+TARGET_PATH = 'data/imdb'
+MAX_FEATURES = 40000
+
+
+def train_simple_ds_model():
+    init_logger()
+    if not not os.path.exists('data/imdb/mapping_rules_labels_t.lib'):
+        minio_connect = MinioConnector()
+        minio_connect.download_dir("datasets/imdb/processed", TARGET_PATH)
+
+    train_df, dev_df, test_df, train_rule_matches_z, dev_rule_matches_z, test_rule_matches_z, imdb_dataset, \
+    mapping_rules_labels_t = \
+        read_train_dev_test(
+            TARGET_PATH)
+    logger.info("Train knn tfidf similarity model")
+
+    X_train = train_df.reviews_preprocessed
+    X_dev = dev_df.reviews_preprocessed
+    X_test = test_df.reviews_preprocessed
+
+    tfidf_values = create_tfidf_values(
+        imdb_dataset.reviews_preprocessed.values, MAX_FEATURES, path_to_cache="tutorials/ImdbDataset/tfidf.lib"
+    )
+
+    train_dataset = TensorDataset(Tensor(tfidf_values[X_train.index].toarray()))
+    dev_dataset = TensorDataset(Tensor(tfidf_values[X_dev.index].toarray()))
+
+    model = LogisticRegressionModel(tfidf_values.shape[1], 2)
+
+    custom_model_config = TrainerConfig(
+        model=model, epochs=35, optimizer_=SGD(model.parameters(), lr=0.1)
+    )
+
+    trainer = NoDenoisingTrainer(
+        model,
+        mapping_rules_labels_t=mapping_rules_labels_t,
+        model_input_x=train_dataset,
+        rule_matches_z=train_rule_matches_z,
+        trainer_config=custom_model_config,
+    )
+    trainer.train()
+
+    tfidf_values_sparse = Tensor(tfidf_values[X_test.index].toarray())
+    tfidf_values_sparse = tfidf_values_sparse.to(custom_model_config.device)
+
+    test_tfidf = TensorDataset(tfidf_values_sparse)
+
+    y_test = Tensor(imdb_dataset.loc[X_test.index, "label_id"].values)
+    y_test = y_test.to(custom_model_config.device)
+    y_test = TensorDataset(y_test)
+
+    clf_report, _ = trainer.test(test_tfidf, y_test)
+    print(clf_report)
+
+
+if __name__ == "__main__":
+    train_simple_ds_model()
diff --git a/examples/trainer/[WIP]_baseline/bert/__init__.py b/examples/trainer/[WIP]_baseline/bert/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/trainer/[WIP]_baseline/bert/bert.ipynb b/examples/trainer/[WIP]_baseline/bert/bert.ipynb
new file mode 100644
index 00000000..c92a6c20
--- /dev/null
+++ b/examples/trainer/[WIP]_baseline/bert/bert.ipynb
@@ -0,0 +1,492 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import sys\n",
+    "import os\n",
+    "import joblib\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "sys.path.append('../../')\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Majority vote + DistillBERT tutorial\n",
+    "\n",
+    "With this tutorial we want to show you how to use a model from huggingface's transformers library within our framework. In order to so, we use the data from our ImdB data creation tutorial.\n",
+    "We make use of \n",
+    "- Majority voting\n",
+    "- DistillBERT classification\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Read data\n",
+    "\n",
+    "We begin by loading the data.\n",
+    "- Make sure you ran the IMDb movie data tutorial before you start. Alternatively, you can download the data with the following command.\n",
+    "- Afterwards we perform a train test split. Observe that we only use a fraction of the data for demonstrational purposes. If you want, you can increase the number of samples."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2021-01-17 00:17:38,695 root         INFO     Initalized logger\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tutorials.baseline.baseline_training_example import read_evaluation_data\n",
+    "\n",
+    "\n",
+    "imdb_dataset, rule_matches_z, mapping_rules_labels_t = read_evaluation_data()\n",
+    "\n",
+    "review_series = imdb_dataset.reviews_preprocessed\n",
+    "label_ids = imdb_dataset.label_id"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "train_data, test_data = train_test_split(\n",
+    "    imdb_dataset[0:100], test_size=0.2, random_state=0\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Preprocess data\n",
+    "\n",
+    "We now preprocess the data to establish a format our Trainer is able to work with. The steps are\n",
+    "- Load the DistillBert tokenizer and tokenize each movie review\n",
+    "- Establish the matrices X, Z and T needed for training\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/andal/.cache/virtual-envs/knodle/lib/python3.7/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)\n",
+      "  return torch._C._cuda_getDeviceCount() > 0\n",
+      "2021-01-17 00:17:41,260 urllib3.connectionpool DEBUG    Starting new HTTPS connection (1): huggingface.co:443\n",
+      "2021-01-17 00:17:41,604 urllib3.connectionpool DEBUG    https://huggingface.co:443 \"HEAD /bert-base-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 0\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from torch.utils.data import TensorDataset\n",
+    "from transformers import DistilBertTokenizer\n",
+    "\n",
+    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
+    "\n",
+    "train_encoding = tokenizer(train_data.reviews_preprocessed.tolist(), return_tensors='pt', padding=True, truncation=True)\n",
+    "train_input_ids = train_encoding['input_ids']\n",
+    "train_attention_mask = train_encoding['attention_mask']\n",
+    "\n",
+    "train_x = TensorDataset(train_input_ids, train_attention_mask)\n",
+    "train_y = TensorDataset(torch.from_numpy(train_data.label_id.values))\n",
+    "train_rule_matches_z = rule_matches_z[train_data.index]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_encoding = tokenizer(test_data.reviews_preprocessed.tolist(), return_tensors='pt', padding=True, truncation=True)\n",
+    "test_input_ids = test_encoding['input_ids']\n",
+    "test_attention_mask = test_encoding['attention_mask']\n",
+    "\n",
+    "test_x = TensorDataset(test_input_ids, test_attention_mask)\n",
+    "test_y = TensorDataset(torch.from_numpy(test_data.label_id.values))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load Model \n",
+    "\n",
+    "After data preparation is finished, we can start with the ML machinery. We need to specify our model, the training configuration and the trainer itself. To ease the start with knodle, we use the same structure as in the popular transformers library.\n",
+    "- Model: We use a distillbert model, as it's working rather well and it's a rather small transformer-based model\n",
+    "- Config: We try to stick close to huggingface's configuration https://huggingface.co/transformers/custom_datasets.html\n",
+    "- Trainer: A custom trainer, which can be found within this folder. It resembles the baseline trainer, just changes the Logistic regression model to a Transformer compatible trainer."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2021-01-17 00:17:42,286 urllib3.connectionpool DEBUG    Starting new HTTPS connection (1): huggingface.co:443\n",
+      "2021-01-17 00:17:42,607 urllib3.connectionpool DEBUG    https://huggingface.co:443 \"HEAD /distilbert-base-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
+      "2021-01-17 00:17:42,640 urllib3.connectionpool DEBUG    Starting new HTTPS connection (1): huggingface.co:443\n",
+      "2021-01-17 00:17:42,979 urllib3.connectionpool DEBUG    https://huggingface.co:443 \"HEAD /distilbert-base-uncased/resolve/main/pytorch_model.bin HTTP/1.1\" 302 0\n",
+      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
+      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "2021-01-17 00:17:46,331 knodle.trainer.ds_model_trainer.ds_model_trainer INFO     Initalized trainer with custom trainer config: {'criterion': <function cross_entropy_with_probs at 0x7f5a62292a70>, 'batch_size': 4, 'epochs': 1, 'optimizer': AdamW (\n",
+      "Parameter Group 0\n",
+      "    betas: (0.9, 0.999)\n",
+      "    correct_bias: True\n",
+      "    eps: 1e-06\n",
+      "    lr: 0.01\n",
+      "    weight_decay: 0.0\n",
+      "), 'output_classes': 2}\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import DistilBertForSequenceClassification, AdamW\n",
+    "from knodle.trainer.config.trainer_config import TrainerConfig\n",
+    "\n",
+    "from tutorials.baseline.bert.majority_bert_trainer import MajorityBertTrainer\n",
+    "\n",
+    "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')\n",
+    "model.train()\n",
+    "\n",
+    "custom_model_config = TrainerConfig(\n",
+    "    model=model, optimizer_= AdamW(model.parameters(), lr=0.01), batch_size=4\n",
+    ")\n",
+    "\n",
+    "trainer = MajorityBertTrainer(\n",
+    "    model,\n",
+    "    mapping_rules_labels_t=mapping_rules_labels_t,\n",
+    "    model_input_x=train_x,\n",
+    "    rule_matches_z=train_rule_matches_z,\n",
+    "    trainer_config=custom_model_config,\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "As we see, we have a standard DistillBERT with an additional classification layer with a binary output, \n",
+    "defining our movie sentiment."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DistilBertForSequenceClassification(\n",
+       "  (distilbert): DistilBertModel(\n",
+       "    (embeddings): Embeddings(\n",
+       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
+       "      (position_embeddings): Embedding(512, 768)\n",
+       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "      (dropout): Dropout(p=0.1, inplace=False)\n",
+       "    )\n",
+       "    (transformer): Transformer(\n",
+       "      (layer): ModuleList(\n",
+       "        (0): TransformerBlock(\n",
+       "          (attention): MultiHeadSelfAttention(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "          (ffn): FFN(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "        )\n",
+       "        (1): TransformerBlock(\n",
+       "          (attention): MultiHeadSelfAttention(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "          (ffn): FFN(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "        )\n",
+       "        (2): TransformerBlock(\n",
+       "          (attention): MultiHeadSelfAttention(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "          (ffn): FFN(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "        )\n",
+       "        (3): TransformerBlock(\n",
+       "          (attention): MultiHeadSelfAttention(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "          (ffn): FFN(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "        )\n",
+       "        (4): TransformerBlock(\n",
+       "          (attention): MultiHeadSelfAttention(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "          (ffn): FFN(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "        )\n",
+       "        (5): TransformerBlock(\n",
+       "          (attention): MultiHeadSelfAttention(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "          (ffn): FFN(\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "          )\n",
+       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n",
+       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
+       "  (dropout): Dropout(p=0.2, inplace=False)\n",
+       ")"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Run training\n",
+    "\n",
+    "In order to run the training procedure, we just need to call the train() method of the trainer."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/andal/devel/knodle/knodle/trainer/utils/denoise.py:20: RuntimeWarning: invalid value encountered in true_divide\n",
+      "  rule_counts_probs = rule_counts / rule_counts.sum(axis=1).reshape(-1, 1)\n",
+      "2021-01-17 00:17:46,511 tutorials.baseline.bert.majority_bert_trainer INFO     ======================================\n",
+      "2021-01-17 00:17:46,511 tutorials.baseline.bert.majority_bert_trainer INFO     Training starts\n",
+      "2021-01-17 00:17:46,512 tutorials.baseline.bert.majority_bert_trainer INFO     ======================================\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1ca579670bdb4092904ad0cf35d866c7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2021-01-17 00:17:46,550 tutorials.baseline.bert.majority_bert_trainer INFO     Epoch: 0\n",
+      "2021-01-17 00:20:55,848 tutorials.baseline.bert.majority_bert_trainer INFO     Epoch loss: 3.2868103981018066\n",
+      "2021-01-17 00:20:55,849 tutorials.baseline.bert.majority_bert_trainer INFO     Epoch Accuracy: 0.5625\n",
+      "2021-01-17 00:20:55,855 tutorials.baseline.bert.majority_bert_trainer INFO     ======================================\n",
+      "2021-01-17 00:20:55,857 tutorials.baseline.bert.majority_bert_trainer INFO     Training done\n",
+      "2021-01-17 00:20:55,863 tutorials.baseline.bert.majority_bert_trainer INFO     ======================================\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Run test set\n",
+    "\n",
+    "Last but not least, we can run the test() method. In case you want to test the properties of multiple test \n",
+    "datasets it's also possible to run this method multiple times."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/andal/.cache/virtual-envs/knodle/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
+      "  'precision', 'predicted', average, warn_for)\n",
+      "2021-01-17 00:21:11,885 knodle.trainer.ds_model_trainer.ds_model_trainer INFO     Accuracy is 0.4\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'0': {'precision': 0.4,\n",
+       "  'recall': 1.0,\n",
+       "  'f1-score': 0.5714285714285715,\n",
+       "  'support': 8},\n",
+       " '1': {'precision': 0.0, 'recall': 0.0, 'f1-score': 0.0, 'support': 12},\n",
+       " 'accuracy': 0.4,\n",
+       " 'macro avg': {'precision': 0.2,\n",
+       "  'recall': 0.5,\n",
+       "  'f1-score': 0.28571428571428575,\n",
+       "  'support': 20},\n",
+       " 'weighted avg': {'precision': 0.16,\n",
+       "  'recall': 0.4,\n",
+       "  'f1-score': 0.2285714285714286,\n",
+       "  'support': 20}}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainer.test(test_features=test_x, test_labels=test_y)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainer.test(test_features=train_x, test_labels=train_y)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/examples/trainer/[WIP]_baseline/bert/requirements.txt b/examples/trainer/[WIP]_baseline/bert/requirements.txt
new file mode 100644
index 00000000..976a2b1f
--- /dev/null
+++ b/examples/trainer/[WIP]_baseline/bert/requirements.txt
@@ -0,0 +1,1 @@
+transformers
diff --git a/examples/trainer/__init__.py b/examples/trainer/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/trainer/preprocessing.py b/examples/trainer/preprocessing.py
new file mode 100644
index 00000000..c96cdbd6
--- /dev/null
+++ b/examples/trainer/preprocessing.py
@@ -0,0 +1,124 @@
+from typing import List, Union, Tuple
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+import numpy as np
+import torch
+from torch.utils.data import TensorDataset
+from transformers import AutoTokenizer
+
+
+def convert_text_to_transformer_input(tokenizer, texts: List[str]) -> TensorDataset:
+    encoding = tokenizer(texts, return_tensors="pt", padding=True, truncation=True)
+    input_ids = encoding.get('input_ids')
+    attention_mask = encoding.get('attention_mask')
+
+    input_values_x = TensorDataset(input_ids, attention_mask)
+
+    return input_values_x
+
+
+def create_bert_input(
+        model: str, train_x: List[str], rule_matches_z: np.array, mapping_rules_labels_t: np.array, train_y: np.array,
+        test_x: List[str], test_y: np.array
+) -> [TensorDataset, np.array, np.array, TensorDataset, TensorDataset, TensorDataset]:
+    """Note that this can also be used for DistillBert and other versions.
+
+    :param tokenizer: An aribrary tokenizer from the transformers library.
+    :return: Data relevant for BERT training.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(model)
+
+    train_ids_mask = convert_text_to_transformer_input(tokenizer, train_x)
+    train_rule_matches_z = rule_matches_z
+    if train_y is not None:
+        train_y = TensorDataset(torch.from_numpy(train_y))
+
+    test_ids_mask = convert_text_to_transformer_input(tokenizer, test_x)
+    test_y = TensorDataset(torch.from_numpy(test_y))
+
+    return (
+        train_ids_mask, train_rule_matches_z, mapping_rules_labels_t, train_y,
+        test_ids_mask, test_y
+    )
+
+
+def create_tfidf_values(text_data: [str], max_features: int = 2000) -> np.array:
+    """Takes text and transforms it.
+
+    :param text_data: List of text strings
+    :param max_features: Maximum number of features for the Vectorizer
+    :return: Tensordataset, holding Tf-Idf values
+    """
+    vectorizer = TfidfVectorizer(max_features=max_features)
+    transformed_data = vectorizer.fit_transform(text_data).toarray().astype(np.float)
+
+    return vectorizer, TensorDataset(torch.from_numpy(transformed_data))
+
+
+def create_tfidf_input(
+        train_x: List[str],
+        rule_matches_z: np.array,
+        mapping_rules_labels_t: np.array,
+        train_y: np.array,
+        test_x: List[str],
+        test_y: np.array,
+        num_features: int = 2000,
+) -> [TensorDataset, np.array, np.array, TensorDataset, TensorDataset, TensorDataset]:
+    """Transform data for trianing purposes.
+
+    :param train_x: List of training texts
+    :param rule_matches_z: Matrix as described in paper
+    :param mapping_rules_labels_t: T matrix
+    :param train_y: Gold labels
+    :param test_x: List of testing texts
+    :param test_y: Gold labels
+    :param num_features: Number of features
+    :return: Data needed for an arbitrary training run based on tf-idf
+    """
+
+    vectorizer = TfidfVectorizer(max_features=num_features)
+    train_x = vectorizer.fit_transform(train_x).toarray().astype(np.float32)
+    train_x = TensorDataset(torch.from_numpy(train_x))
+
+    train_rule_matches_z = rule_matches_z.astype(np.float32)
+    if train_y is not None:
+        train_y = torch.from_numpy(train_y)
+
+    test_x = vectorizer.transform(test_x).toarray().astype(np.float32)
+    test_x = TensorDataset(torch.from_numpy(test_x))
+    test_y = torch.from_numpy(test_y)
+
+    return (
+        train_x,
+        train_rule_matches_z,
+        mapping_rules_labels_t,
+        train_y,
+        test_x,
+        test_y,
+    )
+
+
+def get_tfidf_features(
+        train_data: List, test_data: List = None, dev_data: List = None
+) -> Union[Tuple[np.ndarray, np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray, None]]:
+    """
+    Convert input data to a matrix of TF-IDF features.
+    :param train_data: training samples that are to be encoded with TF-IDF features. Can be given as Series or
+    as DataFrames with specified column number where the sample are stored.
+    :param column_num: optional parameter that is needed to specify in which column of input_data Dataframe the samples
+    are stored
+    :param test_data: if DataFrame/Series with test data is provided
+    :param dev_data: if DataFrame/Series with development data is provided, it will be encoded as well
+    :return: TensorDataset with encoded data
+    """
+    dev_transformed_data, test_transformed_data = None, None
+    vectorizer = TfidfVectorizer()
+
+    train_transformed_data = vectorizer.fit_transform(train_data)
+    if test_data is not None:
+        test_transformed_data = vectorizer.transform(test_data)
+    if dev_data is not None:
+        dev_transformed_data = vectorizer.transform(dev_data)
+
+    return train_transformed_data, test_transformed_data, dev_transformed_data
diff --git a/examples/trainer/simple_auto_trainer/__init__.py b/examples/trainer/simple_auto_trainer/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/trainer/simple_auto_trainer/auto_trainer_tutorial.ipynb b/examples/trainer/simple_auto_trainer/auto_trainer_tutorial.ipynb
new file mode 100644
index 00000000..1f385bd1
--- /dev/null
+++ b/examples/trainer/simple_auto_trainer/auto_trainer_tutorial.ipynb
@@ -0,0 +1,461 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "obvious-chicago",
+   "metadata": {},
+   "source": [
+    "# Auto Trainer Tutorial\n",
+    "\n",
+    "Here we want to show you a really easy example how to use Knodle out-of-the box. This tutorial consists of three main parts:\n",
+    "1. Download data from Knodle Server and load into memory.\n",
+    "2. Initialize Model (DistilBert) and prepare data.\n",
+    "3. Use AutoTrainer for a simple training.\n",
+    "\n",
+    "All the steps are discussed in more detail below."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "responsible-gossip",
+   "metadata": {},
+   "source": [
+    "### Download data\n",
+    " \n",
+    "We will use a preprocessed version of the IMDb dataset. \n",
+    "- In https://github.com/knodle/knodle/examples, you can find a tutorial showing you how the data was preprossed and transformed into the Knodle format. Instead of using the download in the cells below, you can also use this tutorial to create the data yourself.\n",
+    "- The IMDb dataset holds movie reviews. The task is to classify whether a text is a positive or a negative movie review."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "frank-economy",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "\n",
+    "imdb_data_dir = os.path.join(os.getcwd(), \"data\", \"imdb\")\n",
+    "processed_data_dir = os.path.join(imdb_data_dir, \"processed\")\n",
+    "os.makedirs(processed_data_dir, exist_ok=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "silver-venice",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from minio import Minio\n",
+    "from tqdm.auto import tqdm\n",
+    "\n",
+    "client = Minio(\"knodle.dm.univie.ac.at\", secure=False)\n",
+    "files = [\n",
+    "    \"df_train.csv\", \"df_dev.csv\", \"df_test.csv\",\n",
+    "    \"train_rule_matches_z.lib\", \"dev_rule_matches_z.lib\", \"test_rule_matches_z.lib\",\n",
+    "    \"mapping_rules_labels_t.lib\"\n",
+    "]\n",
+    "\n",
+    "for file in tqdm(files):\n",
+    "    client.fget_object(\n",
+    "        bucket_name=\"knodle\",\n",
+    "        object_name=os.path.join(\"datasets/imdb/processed\", file),\n",
+    "        file_path=os.path.join(processed_data_dir, file),\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "natural-slope",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import joblib\n",
+    "import pandas as pd\n",
+    "\n",
+    "df_train = pd.read_csv(os.path.join(processed_data_dir, \"df_train.csv\"))\n",
+    "df_dev = pd.read_csv(os.path.join(processed_data_dir, \"df_dev.csv\"))\n",
+    "df_test = pd.read_csv(os.path.join(processed_data_dir, \"df_test.csv\"))\n",
+    "\n",
+    "mapping_rules_labels_t = joblib.load(os.path.join(processed_data_dir, \"mapping_rules_labels_t.lib\"))\n",
+    "\n",
+    "train_rule_matches_z = joblib.load(os.path.join(processed_data_dir, \"train_rule_matches_z.lib\"))\n",
+    "dev_rule_matches_z = joblib.load(os.path.join(processed_data_dir, \"dev_rule_matches_z.lib\"))\n",
+    "test_rule_matches_z = joblib.load(os.path.join(processed_data_dir, \"test_rule_matches_z.lib\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "operational-presence",
+   "metadata": {},
+   "source": [
+    "### Data description\n",
+    "\n",
+    "We have three splits: train, develop and test split. For each split, there is \n",
+    "- DataFrame, holding text. The training DataFrame only holds text, whereas the development and test set hold text and label.\n",
+    "- a Z matrix, relating instances, or rows in the DataFrame, to rules. \n",
+    "\n",
+    "Again, for more information we refer to the creation of the Dataset https://github.com/knodle/knodle/example."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "unknown-roots",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_train.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ethical-design",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_test.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "unusual-medicine",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(f\"Train Z dimension: {train_rule_matches_z.shape}\")\n",
+    "print(f\"Train avg. matches per sample: {train_rule_matches_z.sum() / train_rule_matches_z.shape[0]}\")\n",
+    "print(f\"Develop avg. matches per sample: {dev_rule_matches_z.sum() / dev_rule_matches_z.shape[0]}\")\n",
+    "print(f\"Test avg. matches per sample: {test_rule_matches_z.sum() / test_rule_matches_z.shape[0]}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "undefined-effort",
+   "metadata": {},
+   "source": [
+    "Here we can can already see the difficulty: On average, each instances has 34 matching rules. Thus the difficulty is to determine which rule or which combination of rules is correct."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "included-reservation",
+   "metadata": {},
+   "source": [
+    "### Preprocess data to DistilBert input\n",
+    "\n",
+    "- Tokenize text. See https://huggingface.co/transformers/ on how to use Transformer-based models.\n",
+    "- Transform data to into the PyTorch tensor format. More specifically, the current Trainers accept TensorDataset, holding a list of tensors. In future, more specialized Datasets might be useful."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "after-thousand",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n",
+    "\n",
+    "import numpy as np\n",
+    "import scipy.sparse as sp\n",
+    "import torch\n",
+    "from torch.utils.data import TensorDataset\n",
+    "\n",
+    "\n",
+    "def convert_text_to_transformer_input(tokenizer, texts: List[str]) -> TensorDataset:\n",
+    "    encoding = tokenizer(texts, return_tensors=\"pt\", padding=True, truncation=True)\n",
+    "    input_ids = encoding.get('input_ids')\n",
+    "    attention_mask = encoding.get('attention_mask')\n",
+    "\n",
+    "    input_values_x = TensorDataset(input_ids, attention_mask)\n",
+    "\n",
+    "    return input_values_x\n",
+    "\n",
+    "\n",
+    "def np_array_to_tensor_dataset(x: np.ndarray) -> TensorDataset:\n",
+    "    if isinstance(x, sp.csr_matrix):\n",
+    "        x = x.toarray()\n",
+    "    x = torch.from_numpy(x)\n",
+    "    x = TensorDataset(x)\n",
+    "    return x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "broke-delay",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer\n",
+    "\n",
+    "\n",
+    "model_name = \"distilbert-base-uncased\"\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+    "\n",
+    "X_train = convert_text_to_transformer_input(tokenizer, df_train[\"sample\"].tolist())\n",
+    "X_dev = convert_text_to_transformer_input(tokenizer, df_dev[\"sample\"].tolist())\n",
+    "X_test = convert_text_to_transformer_input(tokenizer, df_test[\"sample\"].tolist())\n",
+    "\n",
+    "y_dev = np_array_to_tensor_dataset(df_dev['label'].values)\n",
+    "y_test = np_array_to_tensor_dataset(df_test['label'].values)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "tropical-renewal",
+   "metadata": {},
+   "source": [
+    "## Training and evaluation\n",
+    "\n",
+    "In general, Knodle uses the \"Trainer\" data structure to handle training. This is a widely used format in Deep Learning frameworks, used by e.g. PyTorch Lightning (https://pytorch-lightning.readthedocs.io/en/latest/common/trainer.html) or Huggingface's Transformers library (https://huggingface.co/transformers/training.html#trainer). \n",
+    "It takes data and configuration to define training. For each denoising method, a custom Trainer is built. Here, we use the \"MajorityVoteTrainer\". We do so by using the convenience wrapper \"AutoTrainer\". It allows access to different denoising methods by just using a keyword, e.g. \"majority\" in our case.\n",
+    "\n",
+    "The following code shows how to train using the MajorityVoteTrainer. It is a simple baseline, using the following steps:\n",
+    "\n",
+    "1. Restrict data to samples where at least one rule matches\n",
+    "2. Use majority vote to determine the instance labels. If there's no clear winner, randomly choose between labels.\n",
+    "3. Train DistilBert on these weakly formed labels. See https://huggingface.co/transformers/ on how to use Transformer-based models.\n",
+    "\n",
+    "Afterwards, we show two alternative equivalent initializations of the Trainer. Then we show how easy it is to use a different method."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "interracial-extent",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSequenceClassification, AdamW\n",
+    "\n",
+    "from knodle.trainer import AutoTrainer, AutoConfig\n",
+    "\n",
+    "\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
+    "\n",
+    "trainer_type = \"majority\"\n",
+    "custom_model_config = AutoConfig.create_config(\n",
+    "    name=trainer_type,\n",
+    "    optimizer=AdamW,\n",
+    "    lr=1e-4,\n",
+    "    batch_size=16,\n",
+    "    epochs=2,\n",
+    "    filter_non_labelled=True\n",
+    ")\n",
+    "\n",
+    "print(custom_model_config.__dict__)\n",
+    "\n",
+    "trainer = AutoTrainer(\n",
+    "    name=\"majority\",\n",
+    "    model=model,\n",
+    "    mapping_rules_labels_t=mapping_rules_labels_t,\n",
+    "    model_input_x=X_train,\n",
+    "    rule_matches_z=train_rule_matches_z,\n",
+    "    dev_model_input_x=X_dev,\n",
+    "    dev_gold_labels_y=y_dev,\n",
+    "    trainer_config=custom_model_config,\n",
+    ")\n",
+    "\n",
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "solid-church",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eval_dict, _ = trainer.test(X_test, y_test)\n",
+    "print(f\"Accuracy: {eval_dict.get('accuracy')}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "medium-tyler",
+   "metadata": {},
+   "source": [
+    "### Alternative usages\n",
+    "\n",
+    "The following two examples provide exactly the same functionality, just initialize differently."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "overhead-mortality",
+   "metadata": {},
+   "source": [
+    "Here we use the MajorityVoteTrainer explicitly. The code above just provides some convenience."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "hawaiian-concern",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSequenceClassification, AdamW\n",
+    "\n",
+    "from knodle.trainer import MajorityVoteTrainer, MajorityConfig\n",
+    "\n",
+    "\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
+    "\n",
+    "custom_model_config = MajorityConfig(\n",
+    "    optimizer=AdamW,\n",
+    "    lr=1e-4,\n",
+    "    batch_size=16,\n",
+    "    epochs=2,\n",
+    "    filter_non_labelled=True\n",
+    ")\n",
+    "\n",
+    "print(custom_model_config.__dict__)\n",
+    "\n",
+    "trainer = MajorityVoteTrainer(\n",
+    "    model=model,\n",
+    "    mapping_rules_labels_t=mapping_rules_labels_t,\n",
+    "    model_input_x=X_train,\n",
+    "    rule_matches_z=train_rule_matches_z,\n",
+    "    dev_model_input_x=X_dev,\n",
+    "    dev_gold_labels_y=y_dev,\n",
+    "    trainer_config=custom_model_config,\n",
+    ")\n",
+    "\n",
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ordered-kidney",
+   "metadata": {},
+   "source": [
+    "Here, we use a configuration dictionary. This eases the creation of benchmarks as you just have to loop over dictionary values."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "voluntary-disclaimer",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForSequenceClassification, AdamW\n",
+    "\n",
+    "from knodle.trainer import AutoTrainer, AutoConfig\n",
+    "\n",
+    "\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
+    "\n",
+    "config_args = {\n",
+    "    \"name\": \"majority\",\n",
+    "    \"optimizer\": AdamW,\n",
+    "    \"lr\": 1e-4,\n",
+    "    \"batch_size\": 16,\n",
+    "    \"epochs\": 2,\n",
+    "    \"filter_non_labelled\": True\n",
+    "}\n",
+    "custom_model_config = AutoConfig.create_config(**config_args)\n",
+    "\n",
+    "print(custom_model_config.__dict__)\n",
+    "\n",
+    "trainer = AutoTrainer(\n",
+    "    name=config_args[\"name\"],\n",
+    "    model=model,\n",
+    "    mapping_rules_labels_t=mapping_rules_labels_t,\n",
+    "    model_input_x=X_train,\n",
+    "    rule_matches_z=train_rule_matches_z,\n",
+    "    dev_model_input_x=X_dev,\n",
+    "    dev_gold_labels_y=y_dev,\n",
+    "    trainer_config=custom_model_config,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "monthly-telephone",
+   "metadata": {},
+   "source": [
+    "### Use the k-NN - Trainer\n",
+    "\n",
+    "- The following code snippet shows you how easy it is to use a different denoising method.\n",
+    "- The k-NN Trainer takes the k nearest neighbors and adds up the matching rules. Then again, it uses majority voting."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "helpful-mailing",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "config_args[\"name\"] = \"knn\"\n",
+    "config_args[\"k\"] = 3\n",
+    "\n",
+    "custom_model_config = AutoConfig.create_config(**config_args)\n",
+    "\n",
+    "print(custom_model_config.__dict__)\n",
+    "\n",
+    "trainer = AutoTrainer(\n",
+    "    name=config_args[\"name\"],\n",
+    "    model=model,\n",
+    "    mapping_rules_labels_t=mapping_rules_labels_t,\n",
+    "    model_input_x=X_train,\n",
+    "    rule_matches_z=train_rule_matches_z,\n",
+    "    dev_model_input_x=X_dev,\n",
+    "    dev_gold_labels_y=y_dev,\n",
+    "    trainer_config=custom_model_config,\n",
+    ")\n",
+    "\n",
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "treated-mining",
+   "metadata": {},
+   "source": [
+    "## Further readings\n",
+    "\n",
+    "We want to encourage you to head over to our repository\n",
+    "[knodle-experiments](https://github.com/knodle/knodle-experiments)\n",
+    "which adds a new layer of abstraction on top of Knodle, allowing you to easily create full benchmarking setups."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
\ No newline at end of file
diff --git a/examples/trainer/simple_auto_trainer/auto_trainer_tutorial.py b/examples/trainer/simple_auto_trainer/auto_trainer_tutorial.py
new file mode 100644
index 00000000..2d5c4c99
--- /dev/null
+++ b/examples/trainer/simple_auto_trainer/auto_trainer_tutorial.py
@@ -0,0 +1,123 @@
+import os
+from typing import List
+
+from tqdm.auto import tqdm
+
+import joblib
+from minio import Minio
+
+import pandas as pd
+import numpy as np
+import scipy.sparse as sp
+
+import torch
+from torch.utils.data import TensorDataset
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, AdamW
+
+from knodle.trainer import AutoTrainer, AutoConfig
+
+
+# This python script contains rarely any explanation. For more description, we refer to the corresponding
+# jupyter notebook. There the steps are explained in more detail
+
+
+# Define some functions
+
+def convert_text_to_transformer_input(tokenizer, texts: List[str]) -> TensorDataset:
+    encoding = tokenizer(texts, return_tensors="pt", padding=True, truncation=True)
+    input_ids = encoding.get('input_ids')
+    attention_mask = encoding.get('attention_mask')
+
+    input_values_x = TensorDataset(input_ids, attention_mask)
+
+    return input_values_x
+
+
+def np_array_to_tensor_dataset(x: np.ndarray) -> TensorDataset:
+    """
+
+    :rtype: object
+    """
+    if isinstance(x, sp.csr_matrix):
+        x = x.toarray()
+    x = torch.from_numpy(x)
+    x = TensorDataset(x)
+    return x
+
+
+# Define constants
+imdb_data_dir = os.path.join(os.getcwd(), "data", "imdb")
+processed_data_dir = os.path.join(imdb_data_dir, "processed")
+os.makedirs(processed_data_dir, exist_ok=True)
+
+# Download data
+client = Minio("knodle.dm.univie.ac.at", secure=False)
+files = [
+    "df_train.csv", "df_dev.csv", "df_test.csv",
+    "train_rule_matches_z.lib", "dev_rule_matches_z.lib", "test_rule_matches_z.lib",
+    "mapping_rules_labels_t.lib"
+]
+for file in tqdm(files):
+    client.fget_object(
+        bucket_name="knodle",
+        object_name=os.path.join("datasets/imdb/processed", file),
+        file_path=os.path.join(processed_data_dir, file),
+    )
+
+# Load data into memory
+df_train = pd.read_csv(os.path.join(processed_data_dir, "df_train.csv"))
+df_dev = pd.read_csv(os.path.join(processed_data_dir, "df_dev.csv"))
+df_test = pd.read_csv(os.path.join(processed_data_dir, "df_test.csv"))
+
+mapping_rules_labels_t = joblib.load(os.path.join(processed_data_dir, "mapping_rules_labels_t.lib"))
+
+train_rule_matches_z = joblib.load(os.path.join(processed_data_dir, "train_rule_matches_z.lib"))
+dev_rule_matches_z = joblib.load(os.path.join(processed_data_dir, "dev_rule_matches_z.lib"))
+test_rule_matches_z = joblib.load(os.path.join(processed_data_dir, "test_rule_matches_z.lib"))
+
+print(f"Train Z dimension: {train_rule_matches_z.shape}")
+print(f"Train avg. matches per sample: {train_rule_matches_z.sum() / train_rule_matches_z.shape[0]}")
+
+model_name = "distilbert-base-uncased"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+X_train = convert_text_to_transformer_input(tokenizer, df_train["sample"].tolist())
+X_dev = convert_text_to_transformer_input(tokenizer, df_dev["sample"].tolist())
+X_test = convert_text_to_transformer_input(tokenizer, df_test["sample"].tolist())
+
+y_dev = np_array_to_tensor_dataset(df_dev['label'].values)
+y_test = np_array_to_tensor_dataset(df_test['label'].values)
+
+# Load AutoTrainer
+
+model = AutoModelForSequenceClassification.from_pretrained(model_name)
+
+trainer_type = "majority"
+custom_model_config = AutoConfig.create_config(
+    name=trainer_type,
+    optimizer=AdamW,
+    lr=1e-4,
+    batch_size=16,
+    epochs=2,
+    filter_non_labelled=True
+)
+
+print(custom_model_config)
+
+trainer = AutoTrainer(
+    name="majority",
+    model=model,
+    mapping_rules_labels_t=mapping_rules_labels_t,
+    model_input_x=X_train,
+    rule_matches_z=train_rule_matches_z,
+    dev_model_input_x=X_dev,
+    dev_gold_labels_y=y_dev,
+    trainer_config=custom_model_config,
+)
+
+# Run training
+trainer.train()
+
+# Run evaluation
+eval_dict, _ = trainer.test(X_test, y_test)
+print(f"Accuracy: {eval_dict.get('accuracy')}")
diff --git a/examples/trainer/wscrossweigh/__init__.py b/examples/trainer/wscrossweigh/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/trainer/wscrossweigh/requirements.txt b/examples/trainer/wscrossweigh/requirements.txt
new file mode 100644
index 00000000..15a9317d
--- /dev/null
+++ b/examples/trainer/wscrossweigh/requirements.txt
@@ -0,0 +1,15 @@
+numpy~=1.19.2
+scipy~=1.5.3
+joblib~=0.17.0
+pandas~=1.1.5
+pytest~=6.1.2
+torch~=1.7.1
+scikit-learn~=0.21.3
+minio~=7.0.1
+tqdm~=4.50.2
+transformers~=3.4.0
+matplotlib~=3.3.3
+snorkel~=0.9.6
+setuptools~=51.3.3
+python-dotenv~=0.15.0
+spacy~=2.3.2
\ No newline at end of file
diff --git a/examples/trainer/wscrossweigh/wscrossweigh_training_tutorial.py b/examples/trainer/wscrossweigh/wscrossweigh_training_tutorial.py
new file mode 100644
index 00000000..9ef8cf54
--- /dev/null
+++ b/examples/trainer/wscrossweigh/wscrossweigh_training_tutorial.py
@@ -0,0 +1,167 @@
+import argparse
+import os
+import sys
+from typing import Union, Tuple, List
+
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from torch import Tensor, LongTensor
+from torch.optim import Adam
+from torch.utils.data import TensorDataset
+from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, AdamW
+
+from examples.utils import read_train_dev_test
+from knodle.model.logistic_regression_model import LogisticRegressionModel
+from knodle.trainer.wscrossweigh.config import WSCrossWeighDenoisingConfig
+from knodle.trainer.wscrossweigh.wscrossweigh import WSCrossWeighTrainer
+
+
+def train_wscrossweigh(path_to_data: str, num_classes: int) -> None:
+    """
+    We are going to train a BERT classification model using weakly annotated data with additional WSCrossWeigh
+    denoising. The sample weights in WSCrossWeigh will be trained with logistic regression in order to, firstly,
+    reduce the computational effort, and, secondly, demonstrate the ability of the algorithm to use different models
+    for data denoising and classifier training.
+    :param path_to_data: path to the folder where all the input data is stored
+    :param num_classes: number of output classes
+    """
+
+    num_classes = int(num_classes)
+
+    # first, the data is read from the file
+    train_df, dev_df, test_df, z_train_rule_matches, z_test_rule_matches, t_mapping_rules_labels = \
+        read_train_dev_test(path_to_data, if_dev_data=True)
+
+    # we calculate sample weights using logistic regression model (with TF-IDF features) and use the BERT model for final classifier training.
+    train_tfidf_sparse, dev_tfidf_sparse, _ = get_tfidf_features(train_df["sample"].tolist(), dev_df["sample"].tolist())
+
+    train_tfidf = Tensor(train_tfidf_sparse.toarray())
+    train_dataset_tfidf = TensorDataset(train_tfidf)
+
+    # For the BERT training we convert train, dev and test data to BERT encoded features (namely, input indices and attention mask)
+    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    train_input_x_bert = get_bert_encoded_features(train_df["sample"].tolist(), tokenizer)
+    test_labels = TensorDataset(LongTensor(list(test_df.iloc[:, 1])))
+    test_dataset_bert = get_bert_encoded_features(test_df["sample"].tolist(), tokenizer)
+
+    # for some datasets dev set is not provided. If it is not the case, is would be encoded with BERT features
+    # (since it is used only for final training) and passed to a class constructor or to train function
+    if dev_df is not None:
+        dev_dataset_bert = get_bert_encoded_features(dev_df["sample"].tolist(), tokenizer)
+        dev_labels = TensorDataset(LongTensor(list(dev_df.iloc[:, 1])))
+    else:
+        dev_labels, dev_dataset_bert = None, None
+
+    # define the all needed parameters in a dictionary for convenience (can also be directly passed to Trainer/Config)
+    parameters = {
+        "lr": 1e-4, "cw_lr": 0.8, "epochs": 5, "cw_partitions": 2, "cw_folds": 5, "cw_epochs": 2, "weight_rr": 0.7,
+        "samples_start_weights": 4.0
+    }
+    # to have sample weights saved with some specific index in the file name, you can use "caching_suffix" variable
+    caching_suffix = f"dscw_{parameters.get('cw_partitions')}part_" \
+                     f"{parameters.get('cw_folds')}folds_" \
+                     f"{parameters.get('weight_rr')}wrr"
+
+    # define LogReg and BERT models for training sample weights and final classifier correspondingly
+    cw_model = LogisticRegressionModel(train_tfidf.shape[1], num_classes)
+    model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=num_classes)
+
+    # define a custom WSCrossWeigh config. If no custom config is defined, the WSCrossWeighTrainer will use the default
+    # WSCrossWeighDenoisingConfig which is stored in the fold with the WSCrossWeigh trainer
+    custom_wscrossweigh_config = WSCrossWeighDenoisingConfig(
+        # general trainer parameters
+        output_classes=num_classes,
+        filter_non_labelled=False,
+        other_class_id=3,
+        seed=12345,
+        epochs=parameters.get("epochs"),
+        batch_size=16,
+        optimizer=AdamW,
+        lr=parameters.get("lr"),
+        grad_clipping=5,
+        caching_suffix=caching_suffix,
+        saved_models_dir=os.path.join(path_to_data, "trained_models"),  # trained classifier model will be saved after each epoch
+
+        # WSCrossWeigh specific parameters
+        partitions=parameters.get("cw_partitions"),  # number of WSCrossWeigh iterations (= splitting into folds)
+        folds=parameters.get("cw_folds"),  # number of folds train data will be splitted into
+        weight_reducing_rate=parameters.get("weight_rr"),  # sample weights reducing coefficient
+        samples_start_weights=parameters.get("samples_start_weights"),  # the start weight of sample weights
+        cw_epochs=parameters.get("cw_epochs"),  # number of epochs each WSCrossWeigh model is to be trained
+        cw_optimizer=Adam,  # WSCrossWeigh model optimiser
+        cw_lr=parameters.get("cw_lr")  # WSCrossWeigh model lr
+    )
+
+    trainer = WSCrossWeighTrainer(
+        # general Trainer inputs (a more detailed explanation of Knodle inputs is in README)
+        model=model,  # classification model
+        mapping_rules_labels_t=t_mapping_rules_labels,  # t matrix
+        model_input_x=train_input_x_bert,  # x matrix for training the classifier
+        rule_matches_z=z_train_rule_matches,  # z matrix
+        trainer_config=custom_wscrossweigh_config,
+
+        # additional dev set used for classification model evaluation during training
+        dev_model_input_x=dev_dataset_bert,
+        dev_gold_labels_y=dev_labels,
+
+        # WSCrossWeigh specific parameters. If they are not defined, the corresponding main classification parameters
+        # will be used instead (model instead of cw_model etc)
+        cw_model=cw_model,  # model that will be used for WSCrossWeigh weights calculation
+        cw_model_input_x=train_dataset_tfidf,  # x matrix for training the WSCrossWeigh models
+    )
+
+    # the WSCrossWeighTrainer is trained
+    trainer.train()
+    # the trained model is tested on the test set
+    clf_report, _ = trainer.test(test_dataset_bert, test_labels)
+    print(clf_report)
+
+
+def get_bert_encoded_features(input_data: List, tokenizer: DistilBertTokenizer) -> TensorDataset:
+    """
+    Convert input data to BERT encoded features (more details about DistilBert Model could be found at
+    https://huggingface.co/transformers/model_doc/distilbert.html)
+    :param input_data: training/dev/test samples that are to be encoded with BERT features. Can be given as Series or
+    as DataFrames with specified column number where the sample are stored.
+    :param tokenizer: DistilBertTokenizer tokenizer for english from HuggingFace
+    :param column_num: optional parameter that is needed to specify in which column of input_data Dataframe the samples are stored
+    :return: TensorDataset with encoded data
+    """
+    encoding = tokenizer(input_data, return_tensors='pt', padding=True, truncation=True)
+    input_ids = encoding['input_ids']
+    attention_mask = encoding['attention_mask']
+    return TensorDataset(input_ids, attention_mask)
+
+
+def get_tfidf_features(
+        train_data: List, test_data: List = None, dev_data: List = None
+) -> Union[Tuple[np.ndarray, np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray, None]]:
+    """
+    Convert input data to a matrix of TF-IDF features.
+    :param train_data: training samples that are to be encoded with TF-IDF features. Can be given as Series or
+    as DataFrames with specified column number where the sample are stored.
+    :param column_num: optional parameter that is needed to specify in which column of input_data Dataframe the samples
+    are stored
+    :param test_data: if DataFrame/Series with test data is provided
+    :param dev_data: if DataFrame/Series with development data is provided, it will be encoded as well
+    :return: TensorDataset with encoded data
+    """
+    dev_transformed_data, test_transformed_data = None, None
+    vectorizer = TfidfVectorizer()
+
+    train_transformed_data = vectorizer.fit_transform(train_data)
+    if test_data is not None:
+        test_transformed_data = vectorizer.transform(test_data)
+    if dev_data is not None:
+        dev_transformed_data = vectorizer.transform(dev_data)
+
+    return train_transformed_data, test_transformed_data, dev_transformed_data
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]))
+    parser.add_argument("--path_to_data", help="Path to the folder where all input files are stored.")
+    parser.add_argument("--num_classes", help="Number of classes")
+    args = parser.parse_args()
+
+    train_wscrossweigh(args.path_to_data, args.num_classes)
diff --git a/examples/trainer/wscrossweigh/wscrossweigh_training_with_BiLSTM_tutorial.py b/examples/trainer/wscrossweigh/wscrossweigh_training_with_BiLSTM_tutorial.py
new file mode 100644
index 00000000..70e53bfc
--- /dev/null
+++ b/examples/trainer/wscrossweigh/wscrossweigh_training_with_BiLSTM_tutorial.py
@@ -0,0 +1,199 @@
+import argparse
+import os
+import sys
+from typing import Dict
+
+import numpy as np
+import pandas as pd
+import torch
+from torch import Tensor, LongTensor
+from torch.optim import Adam
+from torch.utils.data import TensorDataset
+
+from knodle.evaluation.other_class_metrics import score
+from knodle.model.bidirectional_lstm_model import BidirectionalLSTM
+from knodle.trainer.wscrossweigh.config import WSCrossWeighDenoisingConfig
+from knodle.trainer.wscrossweigh.wscrossweigh import WSCrossWeighTrainer
+from examples.utils import read_train_dev_test, get_samples_list
+
+NUM_CLASSES = 42
+MAXLEN = 50
+SPECIAL_TOKENS = ["<PAD>", "<UNK>"]
+CLASS_WEIGHTS = torch.FloatTensor([2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0,
+                                   2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0,
+                                   2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0])
+
+
+def train_wscrossweigh(
+        path_to_data: str,
+        path_labels: str,
+        path_emb: str,
+        path_sample_weights: str = None,
+) -> None:
+    """ Training the BiLSTM model with WSCrossWeigh denoising algorithm """
+
+    labels2ids = read_labels_from_file(path_labels, "no_relation")
+    word2id, word_embedding_matrix = vocab_and_vectors(path_emb)
+
+    df_train, df_dev, df_test, z_train_rule_matches, z_test_rule_matches, t_mapping_rules_labels = \
+        read_train_dev_test(path_to_data, if_dev_data=True)
+
+    train_input_x = get_samples_features(df_train, word2id, samples_column_num=1)
+
+    dev_dataset = get_samples_features(df_dev, word2id, samples_column_num=1)
+    dev_labels_dataset = TensorDataset(LongTensor(list(df_dev.iloc[:, 4])))
+
+    test_dataset = get_samples_features(df_test, word2id, samples_column_num=1)
+    test_labels_dataset = TensorDataset(LongTensor(list(df_test.iloc[:, 4])))
+
+    os.makedirs(path_sample_weights, exist_ok=True)
+
+    parameters = {
+        "lr": 1e-4,
+        "cw_lr": 0.8,
+        "epochs": 5,
+        "cw_partitions": 2,
+        "cw_folds": 5,
+        "cw_epochs": 2,
+        "weight_rr": 0.7,
+        "samples_start_weights": 4.0
+    }
+
+    model = BidirectionalLSTM(
+        word_embedding_matrix.shape[0], word_embedding_matrix.shape[1], word_embedding_matrix, NUM_CLASSES
+    )
+
+    custom_wscrossweigh_config = WSCrossWeighDenoisingConfig(
+        output_classes=NUM_CLASSES,
+        class_weights=CLASS_WEIGHTS,
+        filter_non_labelled=True,
+        if_set_seed=True,
+        epochs=parameters.get("epochs"),
+        batch_size=16,
+        optimizer=Adam,
+        lr=parameters.get("lr"),
+        grad_clipping=5,
+        partitions=parameters.get("cw_partitions"),
+        folds=parameters.get("cw_folds"),
+        weight_reducing_rate=parameters.get("weight_rr"),
+        samples_start_weights=parameters.get("samples_start_weights")
+    )
+
+    trainer = WSCrossWeighTrainer(
+        model=model,
+        mapping_rules_labels_t=t_mapping_rules_labels,
+        model_input_x=train_input_x,
+        dev_model_input_x=dev_dataset,
+        dev_gold_labels_y=dev_labels_dataset,
+        rule_matches_z=z_train_rule_matches,
+        trainer_config=custom_wscrossweigh_config,
+        evaluation_method="tacred",
+        dev_labels_ids=labels2ids,
+        use_weights=True,
+        run_classifier=True
+    )
+    trainer.train()
+    clf_report, _ = trainer.test(test_dataset, test_labels_dataset)
+    print(clf_report)
+
+
+def get_samples_features(input_data: pd.DataFrame, word2id: dict, samples_column_num: int = None) -> TensorDataset:
+    """ Encodes input samples with glove vectors and returns as a Dataset """
+    enc_input_samples = encode_samples(get_samples_list(input_data, samples_column_num), word2id, MAXLEN)
+    inputs_x_tensor = torch.LongTensor(enc_input_samples)
+    inputs_x_dataset = torch.utils.data.TensorDataset(inputs_x_tensor)
+    return inputs_x_dataset
+
+
+def read_labels_from_file(path_labels: str, negative_label: str) -> dict:
+    """ Reads the labels from the file and encode them with ids """
+    relation2ids = {}
+    with open(path_labels, encoding="UTF-8") as file:
+        for line in file.readlines():
+            relation, relation_enc = line.replace("\n", "").split(",")
+            relation2ids[relation] = int(relation_enc)
+
+    # add no_match label
+    if negative_label:
+        relation2ids[negative_label] = max(list(relation2ids.values())) + 1
+
+    return relation2ids
+
+
+def encode_samples(raw_samples: list, word2id: dict, maxlen: int) -> list:
+    """ This function turns raw text samples into encoded ones using the given word2id dict """
+    enc_input_samples = []
+    for sample in raw_samples:
+        enc_tokens = [word2id.get(token, 1) for token in sample.lstrip().split(" ")]
+        enc_input_samples.append(np.asarray(add_padding(enc_tokens, maxlen), dtype="float32"))
+    return enc_input_samples
+
+
+def add_padding(tokens: list, maxlen: int) -> list:
+    """ Provide padding of the encoded tokens to the maxlen; if length of tokens > maxlen, reduce it to maxlen """
+    padded_tokens = [0] * maxlen
+    for token in range(0, min(len(tokens), maxlen)):
+        padded_tokens[token] = tokens[token]
+    return padded_tokens
+
+
+def vocab_and_vectors(filename: str) -> (dict, np.ndarray):
+    """
+    Reads pretrained word embedding and builds 1) a matrix (words x embedding dim), 2) word to id dictionary
+    :param filename: path to file with pretrained word embeddings
+    :return: word2id, embedding matrix
+    """
+    with open(filename, encoding="UTF-8") as in_file:
+        parts = in_file.readline().strip().split(" ")
+        word_embedding_matrix = np.zeros(
+            (int(parts[0]) + len(SPECIAL_TOKENS), int(parts[1]))
+        )
+        word2id = dict()
+        for idx, token in enumerate(SPECIAL_TOKENS):
+            word2id[token] = idx
+        nextword_id = len(SPECIAL_TOKENS)
+        for line in in_file:
+            parts = line.strip().split(" ")
+            word = parts[0]
+            if word not in word2id:
+                emb = [float(v) for v in parts[1:]]
+                word_embedding_matrix[nextword_id] = emb
+                word2id[word] = nextword_id
+                nextword_id += 1
+    return word2id, word_embedding_matrix
+
+
+# deprecated
+def test(model, trainer, test_features: TensorDataset, test_labels: Tensor, labels2ids: Dict) -> Dict:
+    feature_labels_dataset = TensorDataset(test_features.tensors[0], test_labels)
+    feature_labels_dataloader = trainer._make_dataloader(feature_labels_dataset)
+    model.eval()
+    all_predictions, all_labels = torch.Tensor(), torch.Tensor()
+    for features, labels in feature_labels_dataloader:
+        outputs = model(features)
+        _, predicted = torch.max(outputs, 1)
+        all_predictions = torch.cat([all_predictions, predicted])
+        all_labels = torch.cat([all_labels, labels])
+    predictions_idx, test_labels_idx = (all_predictions.detach().type(torch.IntTensor).tolist(),
+                                        all_labels.detach().type(torch.IntTensor).tolist())
+    idx2labels = dict([(value, key) for key, value in labels2ids.items()])
+    predictions = [idx2labels[p] for p in predictions_idx]
+    test_labels = [idx2labels[p] for p in test_labels_idx]
+    return score(test_labels, predictions, verbose=True)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]))
+    parser.add_argument("--path_to_data", help="Path to the folder where all input files are stored.")
+    parser.add_argument("--path_to_label_ids", help="Path to the file information about labels and their ids is stored")
+    parser.add_argument("--path_to_word_embeddings", help="Path to the file with pretrained Glove embeddings used for "
+                                                          "samples encoding")
+    parser.add_argument("--sample_weights", help="Path to the folder that either sample weights will be saved to"
+                                                 "or will be loaded from")
+    parser.add_argument("--num_classes", help="Number of classes")
+
+    args = parser.parse_args()
+
+    train_wscrossweigh(
+        args.path_to_data, args.path_labels, args.path_emb, args.sample_weights
+    )
diff --git a/examples/utils.py b/examples/utils.py
new file mode 100644
index 00000000..eb77acf1
--- /dev/null
+++ b/examples/utils.py
@@ -0,0 +1,50 @@
+import os
+from typing import Union, Tuple, List
+
+import pandas as pd
+import numpy as np
+from joblib import load, dump
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+
+def read_train_dev_test(
+        target_path: str, if_dev_data: bool = False
+) -> Union[Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, np.ndarray, np.ndarray, np.ndarray],
+           Tuple[pd.DataFrame, None, pd.DataFrame, np.ndarray, np.ndarray, np.ndarray]]:
+    """This function loads the matrices as well as train, test (occasionally, also dev) data from corresponding files"""
+    df_train = load(os.path.join(target_path, 'df_train.lib'))
+    df_test = load(os.path.join(target_path, 'df_test.lib'))
+    z_train_rule_matches = load(os.path.join(target_path, 'train_rule_matches_z.lib'))
+    z_test_rule_matches = load(os.path.join(target_path, 'test_rule_matches_z.lib'))
+    t_mapping_rules_labels = load(os.path.join(target_path, 'mapping_rules_labels_t.lib'))
+
+    if if_dev_data:
+        dev_df = load(os.path.join(target_path, 'df_dev.lib'))
+        return df_train, dev_df, df_test, z_train_rule_matches, z_test_rule_matches, t_mapping_rules_labels
+
+    return df_train, None, df_test, z_train_rule_matches, z_test_rule_matches, t_mapping_rules_labels
+
+
+def get_samples_list(data: Union[pd.Series, pd.DataFrame], column_num: int = None) -> List:
+    """ Extracts the data from the Series/DataFrame and returns it as a list"""
+    column_num = int(column_num)
+    if isinstance(data, pd.Series):
+        return list(data)
+    elif isinstance(data, pd.DataFrame) and column_num is not None:
+        return list(data.iloc[:, column_num])
+    else:
+        raise ValueError(
+            "Please pass input data either as a Series or as a DataFrame with number of the column with samples"
+        )
+
+
+def create_tfidf_values(text_data: [str], max_features, path_to_cache: str = None):
+    if path_to_cache and os.path.exists(path_to_cache):
+        cached_data = load(path_to_cache)
+        if cached_data.shape == text_data.shape:
+            return cached_data
+
+    vectorizer = TfidfVectorizer(max_features=max_features)
+    transformed_data = vectorizer.fit_transform(text_data)
+    dump(transformed_data, path_to_cache)
+    return transformed_data
diff --git a/img/knodle_logo.png b/img/knodle_logo.png
new file mode 100644
index 00000000..93045af0
Binary files /dev/null and b/img/knodle_logo.png differ
diff --git a/img/knodle_logo_1.jpg b/img/knodle_logo_1.jpg
new file mode 100644
index 00000000..5b22c86c
Binary files /dev/null and b/img/knodle_logo_1.jpg differ
diff --git a/img/schema.png b/img/schema.png
new file mode 100644
index 00000000..afef20c4
Binary files /dev/null and b/img/schema.png differ
diff --git a/knodle/__init__.py b/knodle/__init__.py
index c4dfdbc1..fe1579b8 100644
--- a/knodle/__init__.py
+++ b/knodle/__init__.py
@@ -1,6 +1,16 @@
 """knodle - Knowledge infused deep learning framework"""
 
-__version__ = '0.1.0'
-__author__ = 'knodle <alessandro.volpicella@gmail.com>'
+__author__ = "knodle <knodle@cs.univie.ac.at>"
 __all__ = []
+from knodle.version import __version__
 
+import logging
+
+logger = logging.getLogger()
+handler = logging.StreamHandler()
+formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+logger.info("Initalized logger")
diff --git a/knodle/evaluation/__init__.py b/knodle/evaluation/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/evaluation/majority.py b/knodle/evaluation/majority.py
new file mode 100644
index 00000000..b1cbf977
--- /dev/null
+++ b/knodle/evaluation/majority.py
@@ -0,0 +1,35 @@
+from typing import Dict
+
+import numpy as np
+from sklearn.metrics.classification import classification_report
+
+from knodle.transformation.majority import probabilies_to_majority_vote, z_t_matrices_to_majority_vote_probs
+
+
+def majority_sklearn_report(
+        rule_matches_z: np.array, mapping_rules_labels_t: np.array, labels_y: np.array
+) -> Dict:
+    rule_counts_probs = z_t_matrices_to_majority_vote_probs(rule_matches_z, mapping_rules_labels_t)
+
+    kwargs = {"choose_random_label": True}
+    majority_y = np.apply_along_axis(probabilies_to_majority_vote, axis=1, arr=rule_counts_probs, **kwargs)
+
+    sklearn_report = classification_report(labels_y, majority_y, output_dict=True)
+
+    return sklearn_report
+
+
+def sklearn_report_to_knodle_report(sklearn_report: Dict, prefix: str = None):
+    if prefix is None:
+        report = {
+            f"accuracy": sklearn_report["accuracy"],
+            f"macro_f1": sklearn_report["macro avg"]["f1-score"],
+            f"weighted_f1": sklearn_report["weighted avg"]["f1-score"],
+        }
+    else:
+        report = {
+            f"{prefix}accuracy": sklearn_report["accuracy"],
+            f"{prefix}macro_f1": sklearn_report["macro avg"]["f1-score"],
+            f"{prefix}weighted_f1": sklearn_report["weighted avg"]["f1-score"],
+        }
+    return report
diff --git a/knodle/evaluation/other_class_metrics.py b/knodle/evaluation/other_class_metrics.py
new file mode 100644
index 00000000..b8afe8f9
--- /dev/null
+++ b/knodle/evaluation/other_class_metrics.py
@@ -0,0 +1,101 @@
+import logging
+
+from collections import Counter
+from typing import Dict, List
+
+import numpy as np
+
+from knodle.transformation.labels import translate_predictions
+
+logger = logging.getLogger(__name__)
+
+
+def classification_report_other_class(
+        y_true: np.array, y_pred: np.array, ids2labels: Dict, other_class_id: int, verbose: bool = True
+) -> Dict:
+    """ Prepare the batch for the label-based evaluation. """
+    string_prediction, string_gold = translate_predictions(
+        predictions=y_pred, labels=y_true, ids2labels=ids2labels
+    )
+    clf_report = score(key=string_gold, prediction=string_prediction, verbose=verbose,
+                       other_class_label=ids2labels[other_class_id])
+    return clf_report
+
+
+def score(
+        key: List[str], prediction: List[str], verbose: bool, other_class_label: str
+) -> Dict[str, float]:
+    """
+    Computes the precision, recall and f1-score with respect to the other_class label.
+    Other class is treated as a separate negative class not included into the evaluation,
+    i.e. a correctly predicted "other_class" label counts as a TN rather than a TP.
+    Args:
+        key: List with gold string labels for the batch
+        prediction: List with predicted labels for the batch
+        verbose: Whether to output per-relation statistics
+        other_class_label: Label of the class that should be treated as negative.
+    Returns: Decision per sample. Shape: (instances, )
+    """
+    correct_by_relation = Counter()
+    guessed_by_relation = Counter()
+    gold_by_relation = Counter()
+    # Loop over the data to compute a score
+    for row in range(len(key)):
+        gold = key[row]
+        guess = prediction[row]
+        if gold == other_class_label and guess == other_class_label:
+            pass
+        elif gold == other_class_label and guess != other_class_label:
+            guessed_by_relation[guess] += 1
+        elif gold != other_class_label and guess == other_class_label:
+            gold_by_relation[gold] += 1
+        elif gold != other_class_label and guess != other_class_label:
+            guessed_by_relation[guess] += 1
+            gold_by_relation[gold] += 1
+            if gold == guess:
+                correct_by_relation[guess] += 1
+
+    # Print verbose information
+    if verbose:
+        logger.info("Per-relation statistics:")
+        relations = gold_by_relation.keys()
+        longest_relation = 0
+        for relation in sorted(relations):
+            longest_relation = max(len(relation), longest_relation)
+        for relation in sorted(relations):
+            # (compute the score)
+            correct = correct_by_relation[relation]
+            guessed = guessed_by_relation[relation]
+            gold = gold_by_relation[relation]
+            prec = 1.0
+            if guessed > 0:
+                prec = float(correct) / float(guessed)
+            recall = 0.0
+            if gold > 0:
+                recall = float(correct) / float(gold)
+            f1 = 0.0
+            if prec + recall > 0:
+                f1 = 2.0 * prec * recall / (prec + recall)
+            # (print the score)
+            line = f"{relation:{longest_relation}s}\tP: {prec:6.2f}    R: {recall:6.2f}    " \
+                f"F1: {f1:6.2f}    #: {gold:6d}"
+            logger.info(line)
+
+    prec_micro = 1.0
+    if sum(guessed_by_relation.values()) > 0:
+        prec_micro = float(sum(correct_by_relation.values())) / float(sum(guessed_by_relation.values()))
+    recall_micro = 0.0
+    if sum(gold_by_relation.values()) > 0:
+        recall_micro = float(sum(correct_by_relation.values())) / float(sum(gold_by_relation.values()))
+    f1_micro = 0.0
+    if prec_micro + recall_micro > 0.0:
+        f1_micro = 2.0 * prec_micro * recall_micro / (prec_micro + recall_micro)
+
+    # Print the aggregate score
+    logger.info("Final Score:")
+    logger.info("Precision (micro): {:.3%}".format(prec_micro))
+    logger.info("   Recall (micro): {:.3%}".format(recall_micro))
+    logger.info("       F1 (micro): {:.3%}".format(f1_micro))
+    logger.info("\n")
+
+    return {"precision": prec_micro, "recall": recall_micro, "f1": f1_micro}
diff --git a/knodle/evaluation/plotting.py b/knodle/evaluation/plotting.py
new file mode 100644
index 00000000..e50178cf
--- /dev/null
+++ b/knodle/evaluation/plotting.py
@@ -0,0 +1,16 @@
+import matplotlib.pyplot as plt
+
+
+def draw_loss_accuracy_plot(curves: dict) -> None:
+    """ The function creates a plot of 4 curves and displays it"""
+    colors = "bgrcmyk"
+    color_index = 0
+    epochs = range(1, len(next(iter(curves.values()))) + 1)
+
+    for label, value in curves.items():
+        plt.plot(epochs, value, c=colors[color_index], label=label)
+        color_index += 1
+
+    plt.xticks(epochs)
+    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0.)
+    plt.show()
diff --git a/knodle/evaluation/statistics.py b/knodle/evaluation/statistics.py
new file mode 100644
index 00000000..e36f98e4
--- /dev/null
+++ b/knodle/evaluation/statistics.py
@@ -0,0 +1,74 @@
+from typing import Dict, Union, List
+
+import pandas as pd
+import numpy as np
+
+
+def get_y_statistics(y_gold: Union[List, np.ndarray]) -> pd.DataFrame:
+    """Returns a few statistics for a label vector.
+    """
+    y = pd.Series(y_gold)
+    stats_dict = [
+        ["num_classes", y.nunique()],
+        ["num_samples", y_gold.shape[0]],
+        ["classes", y.unique().tolist()],
+        ["class_sums", pd.Series(y_gold).value_counts().to_dict()],
+        ["class_distribution", (pd.Series(y_gold).value_counts() / y_gold.shape[0]).to_dict()]
+    ]
+    stats_dict = pd.DataFrame(stats_dict, columns=["statistic", "value"])
+    return stats_dict
+
+
+def get_z_t_statistics(rule_matches_z: np.array, mapping_rules_labels_t: np.array) -> pd.DataFrame:
+    """Returns a few statistics for a label vector.
+    """
+    hits = np.round(rule_matches_z.sum(axis=0) / rule_matches_z.shape[0], 2)
+    no_match_samples = np.where(hits == 0)[0].sum() / rule_matches_z.sum()
+
+    avg_hits_per_sample = rule_matches_z.sum() / rule_matches_z.shape[0]
+
+    stats_dict = {
+        "num_samples": rule_matches_z.shape[0],
+        "num_classes": mapping_rules_labels_t.shape[1],
+        "num_weak_labels": rule_matches_z.shape[1],
+        "samples_without_match": no_match_samples,
+        "avg_hits_per_sample": avg_hits_per_sample
+    }
+
+    stats_dict = pd.DataFrame(stats_dict, columns=["statistic", "value"])
+    return stats_dict
+
+
+def get_standard_paper_stats(
+        rule_matches_z: np.array, mapping_rules_labels_t: np.array, y_gold: np.ndarray
+) -> pd.DataFrame:
+    """Computes basic statistics relevant for a paper.
+    """
+    if mapping_rules_labels_t.shape[1] == 2:
+        skewdness = round(y_gold.sum() / y_gold.shape[0], 2)
+    else:
+        skewdness = None
+
+    stats_dict = [
+        ["classes", (y_gold.max() + 1).astype(str)],
+        ["train / test samples", f"{rule_matches_z.shape[0]} / {y_gold.shape[0]}"],
+        ["rules", rule_matches_z.shape[1]],
+        ["avg. rule hits", round(rule_matches_z.sum() / rule_matches_z.shape[0], 2)],
+        ["skewdness", skewdness]
+    ]
+    stats_dict = pd.DataFrame(stats_dict, columns=["statistic", "value"])
+    return stats_dict
+
+
+def combine_multiple_paper_stats(dataset_to_df_dict: Dict) -> pd.DataFrame:
+    """Takes a dictionary, having dataset names as keys and DataFrames as values and returns a combined DataFrame.
+    """
+    columns = []
+    values = []
+
+    for dataset, df in dataset_to_df_dict.items():
+        columns = df["statistic"].tolist()
+        values.append([dataset] + df["value"].tolist())
+
+    stats_df = pd.DataFrame(values, columns=["dataset"] + columns)
+    return stats_df
diff --git a/knodle/model/EarlyStopping/__init__.py b/knodle/model/EarlyStopping/__init__.py
new file mode 100644
index 00000000..8164522c
--- /dev/null
+++ b/knodle/model/EarlyStopping/__init__.py
@@ -0,0 +1,51 @@
+import numpy as np
+import torch
+
+
+class EarlyStopping:
+    """Early stops the training if validation loss doesn't improve after a given patience."""
+
+    def __init__(self, patience=7, verbose=False, delta=0, name="checkpoint"):
+        """
+        Args:
+            patience (int): How long to wait after last time validation loss improved.
+                            Default: 7
+            verbose (bool): If True, prints a message for each validation loss improvement.
+                            Default: False
+            delta (float): Minimum change in the monitored quantity to qualify as an improvement.
+                            Default: 0
+        """
+        self.patience = patience
+        self.verbose = verbose
+        self.counter = 0
+        self.best_score = None
+        self.early_stop = False
+        self.val_loss_min = np.Inf
+        self.delta = delta
+        self.name = name
+
+    def __call__(self, val_loss, model):
+
+        score = -val_loss
+
+        if self.best_score is None:
+            self.best_score = score
+            self.save_checkpoint(val_loss, model)
+        elif score < self.best_score + self.delta:
+            self.counter += 1
+            print(f"EarlyStopping counter: {self.counter} out of {self.patience}")
+            if self.counter >= self.patience:
+                self.early_stop = True
+        else:
+            self.best_score = score
+            self.save_checkpoint(val_loss, model)
+            self.counter = 0
+
+    def save_checkpoint(self, val_loss, model):
+        """Saves model when validation loss decrease."""
+        if self.verbose:
+            print(
+                f"Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ..."
+            )
+        torch.save(model.state_dict(), "models/" + self.name + ".pt")
+        self.val_loss_min = val_loss
diff --git a/knodle/model/__init__.py b/knodle/model/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/model/bidirectional_lstm_model.py b/knodle/model/bidirectional_lstm_model.py
new file mode 100644
index 00000000..50dab915
--- /dev/null
+++ b/knodle/model/bidirectional_lstm_model.py
@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+from torch.nn.utils.rnn import pack_padded_sequence
+from torch.nn.utils.rnn import pad_packed_sequence
+
+
+class BidirectionalLSTM(nn.Module):
+    def __init__(
+            self,
+            word_input_dim,
+            word_output_dim,
+            word_embedding_matrix,
+            num_classes,
+            size_factor=200,
+    ):
+        super(BidirectionalLSTM, self).__init__()
+
+        self.word_input_dim = word_input_dim
+        self.word_output_dim = word_output_dim
+        self.word_embedding_matrix = word_embedding_matrix
+        self.size_factor = size_factor
+        self.num_classes = num_classes
+
+        self.word_embedding = nn.Embedding(
+            word_input_dim, word_output_dim, padding_idx=0
+        )
+        self.word_embedding.weight = nn.Parameter(
+            torch.tensor(word_embedding_matrix, dtype=torch.float32)
+        )
+        self.word_embedding.weight.requires_grad = False
+
+        self.type_linear = nn.Linear(20, size_factor * 2)
+
+        self.td_dense = nn.Linear(word_output_dim, size_factor)
+        self.biLSTM = nn.LSTM(
+            size_factor, size_factor, bidirectional=True, batch_first=True
+        )
+
+        self.predict = nn.Linear(size_factor * 2, num_classes)
+        self.init_weights()
+
+    def forward(self, x):
+        word_embeddings = self.word_embedding(x)
+
+        td_dense = self.td_dense(word_embeddings)
+
+        biLSTM, (h_n, c_n) = self.biLSTM(td_dense)
+        self.biLSTM.flatten_parameters()
+
+        final_state = h_n.view(1, 2, x.shape[0], self.size_factor)[-1]
+        h_1, h_2 = final_state[0], final_state[1]  # forward & backward pass
+        concat = torch.cat((h_1, h_2), 1)  # concatenate both states
+        final = self.predict(concat)
+
+        return final
+
+    def init_weights(self):
+        """
+        Here we reproduce Keras default initialization weights to initialize Embeddings/LSTM weights
+        """
+        torch.manual_seed(12345)
+        ih = (
+            param.data for name, param in self.named_parameters() if "weight_ih" in name
+        )
+        hh = (
+            param.data for name, param in self.named_parameters() if "weight_hh" in name
+        )
+        b = (param.data for name, param in self.named_parameters() if "bias" in name)
+        for t in ih:
+            nn.init.xavier_uniform_(t)
+        for t in hh:
+            nn.init.orthogonal_(t)
+        for t in b:
+            nn.init.constant_(t, 0)
diff --git a/knodle/model/logisitc_regression_with_emb_layer.py b/knodle/model/logisitc_regression_with_emb_layer.py
new file mode 100644
index 00000000..7131d94d
--- /dev/null
+++ b/knodle/model/logisitc_regression_with_emb_layer.py
@@ -0,0 +1,32 @@
+import torch
+from torch import nn
+import numpy as np
+
+
+class LogisticRegressionModel(nn.Module):
+    def __init__(
+            self,
+            input_size: int,
+            word_input_dim: int,
+            word_output_dim: int,
+            word_embedding_matrix: np.ndarray,
+            output_classes: int,
+    ):
+        super(LogisticRegressionModel, self).__init__()
+
+        self.word_embedding = nn.Embedding(
+            word_input_dim, word_output_dim, padding_idx=0
+        )
+        self.word_embedding.weight = nn.Parameter(
+            torch.tensor(word_embedding_matrix, dtype=torch.float32)
+        )
+        self.word_embedding.weight.requires_grad = False
+
+        # self.td_dense = nn.Linear(input_size * word_output_dim, size_factor)
+        self.linear = nn.Linear(input_size * word_output_dim, output_classes)
+
+    def forward(self, x):
+        word_embeddings = self.word_embedding(x)
+        word_embeddings = word_embeddings.view(x.shape[0], -1)
+        outputs = self.linear(word_embeddings)
+        return outputs
diff --git a/knodle/model/logistic_regression_model.py b/knodle/model/logistic_regression_model.py
new file mode 100644
index 00000000..94b7e40e
--- /dev/null
+++ b/knodle/model/logistic_regression_model.py
@@ -0,0 +1,13 @@
+import torch
+from torch import nn
+
+
+class LogisticRegressionModel(nn.Module):
+    def __init__(self, input_dim: int, output_classes: int):
+        super(LogisticRegressionModel, self).__init__()
+        self.linear = torch.nn.Linear(input_dim, output_classes)
+
+    def forward(self, x):
+        x = x.float()
+        outputs = self.linear(x)
+        return outputs
diff --git a/knodle/sum.py b/knodle/sum.py
deleted file mode 100644
index ffa9bfa9..00000000
--- a/knodle/sum.py
+++ /dev/null
@@ -1,19 +0,0 @@
-
-def test_sum_function(alpha: int, b: int) -> int:
-    """
-    Function for test purpose
-    :param a:
-    :param b:
-    :return:
-    """
-    return alpha + b
-
-
-def test_minus_function(a: int, b: int) -> int:
-    """
-    Function for test purpose
-    :param a:
-    :param b:
-    :return:
-    """
-    return a - b
diff --git a/knodle/trainer/__init__.py b/knodle/trainer/__init__.py
new file mode 100644
index 00000000..53a7f141
--- /dev/null
+++ b/knodle/trainer/__init__.py
@@ -0,0 +1,8 @@
+from knodle.trainer.config import TrainerConfig
+
+from knodle.trainer.baseline.majority import MajorityVoteTrainer, MajorityConfig
+from knodle.trainer.knn_denoising.knn import KnnDenoisingTrainer, KNNConfig
+from knodle.trainer.snorkel.snorkel import SnorkelKNNDenoisingTrainer, SnorkelTrainer, SnorkelConfig, SnorkelKNNConfig
+
+from knodle.trainer.auto_trainer import AutoTrainer
+from knodle.trainer.auto_config import AutoConfig
\ No newline at end of file
diff --git a/knodle/trainer/auto_config.py b/knodle/trainer/auto_config.py
new file mode 100644
index 00000000..fdb37c4c
--- /dev/null
+++ b/knodle/trainer/auto_config.py
@@ -0,0 +1,31 @@
+from typing import Callable
+
+from knodle.trainer.config import TrainerConfig
+
+
+class AutoConfig:
+    """ The factory class for creating Config classes of training executors
+    See See https://medium.com/@geoffreykoh/implementing-the-factory-
+    pattern-via-dynamic-registry-and-python-decorators-479fc1537bbe
+    """
+
+    registry = {}
+    """ Internal registry for available trainers """
+
+    def __init__(self, name, **kwargs):
+        self.config = self.create_trainer(name, **kwargs)
+
+    @classmethod
+    def create_config(cls, name: str, **kwargs) -> TrainerConfig:
+
+        exec_class = cls.registry[name]
+        executor = exec_class(**kwargs)
+        return executor
+
+    @classmethod
+    def register(cls, name: str) -> Callable:
+        def inner_wrapper(wrapped_class: TrainerConfig) -> Callable:
+            cls.registry[name] = wrapped_class
+            return wrapped_class
+
+        return inner_wrapper
diff --git a/knodle/trainer/auto_trainer.py b/knodle/trainer/auto_trainer.py
new file mode 100644
index 00000000..e46b630e
--- /dev/null
+++ b/knodle/trainer/auto_trainer.py
@@ -0,0 +1,39 @@
+from typing import Callable
+from torch.utils.data import TensorDataset
+
+from knodle.trainer.trainer import Trainer
+
+
+class AutoTrainer:
+    """ The factory class for creating training executors
+    See See https://medium.com/@geoffreykoh/implementing-the-factory-
+    pattern-via-dynamic-registry-and-python-decorators-479fc1537bbe
+    """
+
+    registry = {}
+    """ Internal registry for available trainers """
+
+    def __init__(self, name, **kwargs):
+        self.trainer = self.create_trainer(name, **kwargs)
+
+    def train(self):
+        return self.trainer.train()
+
+    def test(self, test_features: TensorDataset, test_labels: TensorDataset):
+        return self.trainer.test(test_features, test_labels)
+
+    @classmethod
+    def create_trainer(cls, name: str, **kwargs) -> Trainer:
+        """ Factory command to create the executor """
+
+        exec_class = cls.registry[name]
+        executor = exec_class(**kwargs)
+        return executor
+
+    @classmethod
+    def register(cls, name: str) -> Callable:
+        def inner_wrapper(wrapped_class: Trainer) -> Callable:
+            cls.registry[name] = wrapped_class
+            return wrapped_class
+
+        return inner_wrapper
diff --git a/knodle/trainer/baseline/__init__.py b/knodle/trainer/baseline/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/trainer/baseline/config.py b/knodle/trainer/baseline/config.py
new file mode 100644
index 00000000..20fd3d39
--- /dev/null
+++ b/knodle/trainer/baseline/config.py
@@ -0,0 +1,14 @@
+from knodle.trainer.auto_config import AutoConfig
+from knodle.trainer.config import BaseTrainerConfig
+
+
+@AutoConfig.register("majority")
+class MajorityConfig(BaseTrainerConfig):
+    def __init__(
+            self,
+            use_probabilistic_labels: bool = True,
+            **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.use_probabilistic_labels = use_probabilistic_labels
+        self.probability_threshold = None
diff --git a/knodle/trainer/baseline/majority.py b/knodle/trainer/baseline/majority.py
new file mode 100644
index 00000000..26c777b1
--- /dev/null
+++ b/knodle/trainer/baseline/majority.py
@@ -0,0 +1,70 @@
+import logging
+
+import numpy as np
+
+import torch.nn as nn
+from torch.optim import SGD
+from torch.utils.data import TensorDataset
+
+from knodle.transformation.majority import input_to_majority_vote_input
+from knodle.transformation.torch_input import input_labels_to_tensordataset
+
+from knodle.trainer.trainer import BaseTrainer
+from knodle.trainer.auto_trainer import AutoTrainer
+from knodle.trainer.baseline.config import MajorityConfig
+from knodle.transformation.filter import filter_unconclusive_probabilities
+
+logger = logging.getLogger(__name__)
+
+
+@AutoTrainer.register('majority')
+class MajorityVoteTrainer(BaseTrainer):
+    """
+    The baseline class implements a baseline model for labeling data with weak supervision.
+        A simple majority vote is used for this purpose.
+    """
+
+    def __init__(
+            self,
+            model: nn.Module,
+            mapping_rules_labels_t: np.ndarray,
+            model_input_x: TensorDataset,
+            rule_matches_z: np.ndarray,
+            trainer_config: MajorityConfig = None,
+            **kwargs
+    ):
+        if trainer_config is None:
+            trainer_config = MajorityConfig(optimizer=SGD, lr=0.001)
+        super().__init__(
+            model, mapping_rules_labels_t, model_input_x, rule_matches_z, trainer_config=trainer_config, **kwargs
+        )
+
+    def train(
+            self,
+            model_input_x: TensorDataset = None, rule_matches_z: np.ndarray = None,
+            dev_model_input_x: TensorDataset = None, dev_gold_labels_y: TensorDataset = None,
+            probability_threshold: float = None
+    ):
+        """
+        This function gets final labels with a majority vote approach and trains the provided model.
+        """
+        self._load_train_params(model_input_x, rule_matches_z, dev_model_input_x, dev_gold_labels_y)
+
+        # initialise optimizer
+        self.trainer_config.optimizer = self.initialise_optimizer()
+
+        model_input_x, label_probs = input_to_majority_vote_input(
+            self.model_input_x, self.rule_matches_z, self.mapping_rules_labels_t,
+            filter_non_labelled=self.trainer_config.filter_non_labelled,
+            other_class_id=self.trainer_config.other_class_id
+        )
+
+        if self.trainer_config.probability_threshold is not None:
+            model_input_x, label_probs = filter_unconclusive_probabilities(
+                model_input_x, label_probs, probability_threshold=self.trainer_config.probability_threshold
+            )
+
+        feature_label_dataset = input_labels_to_tensordataset(model_input_x, label_probs)
+        feature_label_dataloader = self._make_dataloader(feature_label_dataset)
+
+        self._train_loop(feature_label_dataloader)
diff --git a/knodle/trainer/config.py b/knodle/trainer/config.py
new file mode 100644
index 00000000..253be0d4
--- /dev/null
+++ b/knodle/trainer/config.py
@@ -0,0 +1,101 @@
+import pathlib
+from typing import Callable, Dict
+import os
+import logging
+
+from snorkel.classification import cross_entropy_with_probs
+
+import torch
+from torch import Tensor
+from torch.optim.optimizer import Optimizer
+
+from knodle.trainer.utils.utils import check_and_return_device, set_seed
+
+logger = logging.getLogger(__name__)
+
+
+class TrainerConfig:
+    def __init__(
+            self,
+            criterion: Callable[[Tensor, Tensor], float] = cross_entropy_with_probs,
+            batch_size: int = 32,
+            optimizer: Optimizer = None,
+            lr: int = 0.01,
+            output_classes: int = 2,
+            class_weights: Tensor = None,
+            epochs: int = 3,
+            seed: int = 42,
+            grad_clipping: int = None,
+            device: str = None,
+            caching_folder: str = None,
+            caching_suffix: str = "",
+            saved_models_dir: str = None
+    ):
+        self.seed = seed
+        set_seed(seed)
+
+        # create directory where saved models will be stored
+        self.saved_models_dir = saved_models_dir
+        if self.saved_models_dir is not None:
+            os.makedirs(self.saved_models_dir, exist_ok=True)
+
+        self.criterion = criterion
+        self.lr = lr
+        self.batch_size = batch_size
+        self.output_classes = output_classes
+        self.grad_clipping = grad_clipping
+        self.device = torch.device(device) if device is not None else check_and_return_device()
+        logger.info(f"Model will be trained on {self.device}")
+
+        self.caching_suffix = caching_suffix
+        if caching_folder is not None:
+            self.caching_folder = caching_folder
+        else:
+            self.caching_folder = os.path.join(pathlib.Path().absolute(), "cache")
+        logger.info(f"The cache will be saved to {self.caching_folder} folder")
+
+        if epochs <= 0:
+            raise ValueError("Epochs needs to be positive")
+        self.epochs = epochs
+
+        if optimizer is None:
+            raise ValueError("An optimizer needs to be provided")
+        else:
+            self.optimizer = optimizer
+
+        if class_weights is None:
+            self.class_weights = torch.tensor([1.0] * self.output_classes)
+        else:
+            if len(class_weights) != self.output_classes:
+                raise Exception("Wrong class sample_weights initialisation!")
+            self.class_weights = class_weights
+
+
+class BaseTrainerConfig(TrainerConfig):
+    def __init__(
+            self,
+            filter_non_labelled: bool = True,
+            other_class_id: int = None,
+            evaluate_with_other_class: bool = False,
+            ids2labels: Dict = None,
+            **kwargs
+    ):
+        """ Additionally provided parameters needed for handling the cases where there are data samples with no rule
+         matched (filtering OR introducing the other class + training&evaluation with other class) """
+        super().__init__(**kwargs)
+        self.filter_non_labelled = filter_non_labelled
+        self.other_class_id = other_class_id
+        self.evaluate_with_other_class = evaluate_with_other_class
+        self.ids2labels = ids2labels
+
+        if self.other_class_id is not None and self.filter_non_labelled:
+            raise ValueError("You can either filter samples with no weak labels or add them to 'other_class_id'")
+
+        logger.debug(f"{self.evaluate_with_other_class} and {self.ids2labels}")
+        if self.evaluate_with_other_class and self.ids2labels is None:
+            # check if the selected evaluation type is valid
+            logging.warning(
+                "Labels to label ids correspondence is needed to make other_class specific evaluation. Since it is "
+                "absent now, the standard sklearn metrics will be calculated instead."
+            )
+            self.evaluate_with_other_class = False
diff --git a/knodle/trainer/knn_denoising/__init__.py b/knodle/trainer/knn_denoising/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/trainer/knn_denoising/config.py b/knodle/trainer/knn_denoising/config.py
new file mode 100644
index 00000000..0a51f623
--- /dev/null
+++ b/knodle/trainer/knn_denoising/config.py
@@ -0,0 +1,57 @@
+import os
+
+from knodle.trainer.baseline.config import MajorityConfig
+from knodle.trainer.auto_config import AutoConfig
+
+
+@AutoConfig.register("knn")
+class KNNConfig(MajorityConfig):
+    def __init__(
+            self,
+            k: int = None,
+            radius: float = None,
+            weighted_knn_activation: bool = False,
+            use_approximation: bool = False,
+            activate_no_match_instances: bool = True,
+            n_jobs_for_index: int = 4,
+            **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.k = k
+        self.radius = radius
+        self.weighted_knn_activation = weighted_knn_activation
+        self.use_approximation = use_approximation
+        self.activate_no_match_instances = activate_no_match_instances
+        self.n_jobs = n_jobs_for_index
+
+        if self.k is not None and self.radius is not None:
+            raise RuntimeError(
+                "The Knn trainer can either use the radius or the number of "
+                "neighbours to denoise by neighborhood activation"
+            )
+
+        if self.k is None and self.use_approximation:
+            raise RuntimeError(
+                "The Knn trainer can only use the radius for exact neighbor search "
+                "Distance-based selection is currently unavailable for approximate NN."
+            )
+
+        # Currently impossible, though can (should?) be done in the future.
+        if not self.use_approximation and not self.activate_no_match_instances:
+            raise RuntimeError(
+                "The Knn trainer with exact neighbor selection always uses all of the instances. "
+                "Either 'activate_no_match_instances' or 'use_approximation' has to be set to True."
+            )
+
+    def get_cache_file(self):
+        nn_type = "ann" if self.use_approximation else "knn"
+        file_tags = f"{self.k}_{nn_type}"
+        if self.caching_suffix:
+            file_tags += f"_{self.caching_suffix}"
+
+        cache_file = os.path.join(
+            self.caching_folder,
+            f"denoised_rule_matches_z_{file_tags}.lib"
+        )
+
+        return cache_file
diff --git a/knodle/trainer/knn_denoising/knn.py b/knodle/trainer/knn_denoising/knn.py
new file mode 100644
index 00000000..17668e5d
--- /dev/null
+++ b/knodle/trainer/knn_denoising/knn.py
@@ -0,0 +1,146 @@
+import os
+import logging
+
+import joblib
+import numpy as np
+
+from torch.optim import SGD
+from torch.utils.data import TensorDataset
+
+from scipy.sparse import csr_matrix
+from sklearn.neighbors import NearestNeighbors
+from annoy import AnnoyIndex
+
+from knodle.transformation.majority import input_to_majority_vote_input
+from knodle.transformation.torch_input import input_labels_to_tensordataset
+
+from knodle.trainer.auto_trainer import AutoTrainer
+from knodle.trainer.baseline.majority import MajorityVoteTrainer
+from knodle.trainer.knn_denoising.config import KNNConfig
+from knodle.trainer.utils.denoise import activate_neighbors
+
+logger = logging.getLogger(__name__)
+
+
+@AutoTrainer.register('knn')
+class KnnDenoisingTrainer(MajorityVoteTrainer):
+    def __init__(
+            self,
+            knn_feature_matrix: np.ndarray = None,
+            **kwargs
+    ):
+        if kwargs.get("trainer_config") is None:
+            kwargs["trainer_config"] = KNNConfig(optimizer=SGD, lr=0.001)
+        super().__init__(**kwargs)
+
+        if knn_feature_matrix is None:
+            self.knn_feature_matrix = csr_matrix(self.model_input_x.tensors[0].numpy())
+        else:
+            self.knn_feature_matrix = knn_feature_matrix
+
+    def train(
+            self,
+            model_input_x: TensorDataset = None, rule_matches_z: np.ndarray = None,
+            dev_model_input_x: TensorDataset = None, dev_gold_labels_y: TensorDataset = None
+    ):
+        self._load_train_params(model_input_x, rule_matches_z, dev_model_input_x, dev_gold_labels_y)
+
+        # initialise optimizer
+        self.trainer_config.optimizer = self.initialise_optimizer()
+
+        self.rule_matches_z = self.rule_matches_z.astype(np.int8)
+        self.mapping_rules_labels_t = self.mapping_rules_labels_t.astype(np.int8)
+
+        self._knn_denoise_rule_matches()
+
+        model_input_x, label_probs = input_to_majority_vote_input(
+            self.model_input_x, self.rule_matches_z, self.mapping_rules_labels_t.astype(np.int64),
+            filter_non_labelled=self.trainer_config.filter_non_labelled,
+            other_class_id=self.trainer_config.other_class_id
+        )
+
+        feature_label_dataset = input_labels_to_tensordataset(model_input_x, label_probs)
+        feature_label_dataloader = self._make_dataloader(feature_label_dataset)
+
+        self._train_loop(feature_label_dataloader)
+
+    def _knn_denoise_rule_matches(self) -> np.ndarray:
+        """
+        Denoises the applied weak supervision source.
+        Args:
+            rule_matches_z: Matrix with all applied weak supervision sources. Shape: (Instances x Rules)
+        Returns: Denoised / Improved applied labeling function matrix. Shape: (Instances x Rules)
+        """
+        k = self.trainer_config.k
+        if k == 1:
+            return self.rule_matches_z
+
+        # load cached data, if available
+        if self.trainer_config.caching_folder:
+            cache_file = self.trainer_config.get_cache_file()
+            if os.path.isfile(cache_file):
+                logger.info(f"Loaded knn matrix from cache: {cache_file}")
+                return joblib.load(cache_file)
+
+        logger.info(f"Start denoising labeling functions with k: {k}.")
+
+        # ignore zero-match rows for knn construction & activation
+        if self.trainer_config.activate_no_match_instances:
+            ignore = np.zeros((self.knn_feature_matrix.shape[0],), dtype=np.bool)
+        else:
+            ignore = self.rule_matches_z.sum(-1) == 0
+
+        # Set up data structure, to quickly find nearest neighbors
+        if self.trainer_config.use_approximation:
+            # use annoy fast ANN
+            if k is not None:
+                knn_matrix_shape = self.knn_feature_matrix.shape
+
+                logger.info("Creating annoy index...")
+                t = AnnoyIndex(knn_matrix_shape[1], 'dot')
+                for i, v in enumerate(self.knn_feature_matrix):
+                    if not ignore[i]:
+                        t.add_item(i, v)
+
+                t.build(10, n_jobs=self.trainer_config.n_jobs)
+
+                self.knn_feature_matrix = None
+
+                logger.info("Retrieving neighbor indices...")
+                indices = (         # make a generator: no memory is allocated at this moment
+                    np.array(t.get_nns_by_item(i, k, search_k=-1, include_distances=False))
+                    if not ignore[i] else np.array([])
+                    for i in range(knn_matrix_shape[0])
+                )
+            else:
+                # possible radius implementation; delete error in config then
+                pass
+        else:
+            # use standard precise kNN
+            if k is not None:
+                logger.info("Creating NN index...")
+                neighbors = NearestNeighbors(n_neighbors=k, n_jobs=self.trainer_config.n_jobs)\
+                    .fit(self.knn_feature_matrix)
+                logger.info("Retrieving neighbor indices...")
+                indices = neighbors.kneighbors(self.knn_feature_matrix, n_neighbors=k, return_distance=False)
+            else:
+                logger.info("Creating NN index...")
+                neighbors = NearestNeighbors(radius=self.trainer_config.radius, n_jobs=self.trainer_config.n_jobs)\
+                    .fit(self.knn_feature_matrix)
+                logger.info("Retrieving neighbor indices...")
+                indices = neighbors.radius_neighbors(self.knn_feature_matrix, return_distance=False)
+
+        # activate matches.
+        logger.info("Activating neighbors...")
+        self.rule_matches_z = activate_neighbors(self.rule_matches_z, indices)
+
+        # save data for caching
+        if self.trainer_config.caching_folder:
+            os.makedirs(self.trainer_config.caching_folder, exist_ok=True)
+            joblib.dump(self.rule_matches_z, cache_file)
+
+        return self.rule_matches_z
+
+    def print_step_update(self, step: int, max_steps: int):
+        if step % 40 == 0 and not step == 0:
+            logger.info(f"  Batch {step}  of  {max_steps}.")
diff --git a/knodle/trainer/snorkel/__init__.py b/knodle/trainer/snorkel/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/trainer/snorkel/config.py b/knodle/trainer/snorkel/config.py
new file mode 100644
index 00000000..4b5eea5b
--- /dev/null
+++ b/knodle/trainer/snorkel/config.py
@@ -0,0 +1,24 @@
+from knodle.trainer.baseline.config import MajorityConfig
+from knodle.trainer.knn_denoising.config import KNNConfig
+
+from knodle.trainer.auto_config import AutoConfig
+
+
+@AutoConfig.register("snorkel")
+class SnorkelConfig(MajorityConfig):
+    def __init__(
+            self,
+            label_model_num_epochs: int = 5000,
+            label_model_log_freq: int = 500,
+            **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.label_model_num_epochs = label_model_num_epochs
+        self.label_model_log_freq = label_model_log_freq
+
+
+@AutoConfig.register("snorkel_knn")
+class SnorkelKNNConfig(SnorkelConfig, KNNConfig):
+    def __init__(self, **kwargs):
+        # use all config parameters in SnorkelConfig and KNNConfig
+        super().__init__(**kwargs)
diff --git a/knodle/trainer/snorkel/snorkel.py b/knodle/trainer/snorkel/snorkel.py
new file mode 100644
index 00000000..882693c0
--- /dev/null
+++ b/knodle/trainer/snorkel/snorkel.py
@@ -0,0 +1,82 @@
+import numpy as np
+from snorkel.labeling.model import LabelModel
+
+from torch.optim import SGD
+from torch.utils.data import TensorDataset
+
+from knodle.transformation.torch_input import input_labels_to_tensordataset
+
+from knodle.trainer.auto_trainer import AutoTrainer
+from knodle.trainer.baseline.majority import MajorityVoteTrainer
+from knodle.trainer.knn_denoising.knn import KnnDenoisingTrainer
+
+from knodle.trainer.snorkel.config import SnorkelConfig, SnorkelKNNConfig
+from knodle.trainer.snorkel.utils import z_t_matrix_to_snorkel_matrix
+
+
+@AutoTrainer.register('snorkel')
+class SnorkelTrainer(MajorityVoteTrainer):
+    def __init__(self, **kwargs):
+        if kwargs.get("trainer_config", None) is None:
+            kwargs["trainer_config"] = SnorkelConfig(optimizer=SGD, lr=0.001)
+        super().__init__(**kwargs)
+
+    def _snorkel_denoising(self, model_input_x, rule_matches_z):
+
+        # initialise optimizer
+        self.trainer_config.optimizer = self.initialise_optimizer()
+
+        non_zero_indices = np.where(rule_matches_z.sum(axis=1) != 0)[0]
+        rule_matches_z = rule_matches_z[non_zero_indices]
+        tensors = list(model_input_x.tensors)
+        for i in range(len(tensors)):
+            tensors[i] = tensors[i][non_zero_indices]
+
+        model_input_x = TensorDataset(*tensors)
+
+        # create Snorkel matrix and train LabelModel
+        L_train = z_t_matrix_to_snorkel_matrix(rule_matches_z, self.mapping_rules_labels_t)
+
+        label_model = LabelModel(cardinality=self.mapping_rules_labels_t.shape[1], verbose=True)
+        label_model.fit(
+            L_train,
+            n_epochs=self.trainer_config.label_model_num_epochs,
+            log_freq=self.trainer_config.label_model_log_freq,
+            seed=self.trainer_config.seed
+        )
+        label_probs = label_model.predict_proba(L_train)
+        return model_input_x, label_probs
+
+    def train(
+            self,
+            model_input_x: TensorDataset = None, rule_matches_z: np.ndarray = None,
+            dev_model_input_x: TensorDataset = None, dev_gold_labels_y: TensorDataset = None
+    ):
+        self._load_train_params(model_input_x, rule_matches_z, dev_model_input_x, dev_gold_labels_y)
+
+        model_input_x, label_probs = self._snorkel_denoising(self.model_input_x, self.rule_matches_z)
+
+        # Standard training
+        feature_label_dataset = input_labels_to_tensordataset(model_input_x, label_probs)
+        feature_label_dataloader = self._make_dataloader(feature_label_dataset)
+
+        self._train_loop(feature_label_dataloader)
+
+
+@AutoTrainer.register('snorkel_knn')
+class SnorkelKNNDenoisingTrainer(SnorkelTrainer, KnnDenoisingTrainer):
+    def __init__(self, **kwargs):
+        if kwargs.get("trainer_config", None) is None:
+            kwargs["trainer_config"] = SnorkelKNNConfig(optimizer=SGD, lr=0.001)
+        super().__init__(**kwargs)
+
+    def train(self):
+        # Snorkel denoising
+        denoised_rule_matches_z = self._knn_denoise_rule_matches()
+        model_input_x, label_probs = self._snorkel_denoising(self.model_input_x, denoised_rule_matches_z)
+
+        # Standard training
+        feature_label_dataset = input_labels_to_tensordataset(model_input_x, label_probs)
+        feature_label_dataloader = self._make_dataloader(feature_label_dataset)
+
+        self._train_loop(feature_label_dataloader)
diff --git a/knodle/trainer/snorkel/utils.py b/knodle/trainer/snorkel/utils.py
new file mode 100644
index 00000000..ba1e49b7
--- /dev/null
+++ b/knodle/trainer/snorkel/utils.py
@@ -0,0 +1,16 @@
+import numpy as np
+import scipy.sparse as sp
+
+
+def z_t_matrix_to_snorkel_matrix(rule_matches_z: np.ndarray, mapping_rules_labels_t: np.ndarray) -> np.ndarray:
+    snorkel_matrix = -1 * np.ones(rule_matches_z.shape)
+
+    if isinstance(rule_matches_z, sp.csr_matrix):
+        rule_matches_z = rule_matches_z.toarray()
+
+    z_to_t = np.argmax(mapping_rules_labels_t, axis=-1)
+    for i in range(rule_matches_z.shape[0]):
+        non_zero_idx = np.where(rule_matches_z[i] != 0)[0]
+        snorkel_matrix[i, non_zero_idx] = z_to_t[non_zero_idx]
+
+    return snorkel_matrix
\ No newline at end of file
diff --git a/knodle/trainer/trainer.py b/knodle/trainer/trainer.py
new file mode 100644
index 00000000..6f23628b
--- /dev/null
+++ b/knodle/trainer/trainer.py
@@ -0,0 +1,261 @@
+import os
+import logging
+from typing import Union, Dict, Tuple
+from tqdm.auto import tqdm
+from abc import ABC, abstractmethod
+
+import numpy as np
+from sklearn.metrics import classification_report
+
+import torch
+from torch import Tensor
+from torch.nn import Module
+from torch.utils.data import TensorDataset, DataLoader
+import torch.nn.functional as F
+
+from knodle.evaluation.other_class_metrics import classification_report_other_class
+from knodle.transformation.torch_input import input_labels_to_tensordataset
+from knodle.evaluation.plotting import draw_loss_accuracy_plot
+
+from knodle.trainer.config import BaseTrainerConfig
+from knodle.trainer.utils.utils import log_section, accuracy_of_probs
+
+logger = logging.getLogger(__name__)
+
+
+class Trainer(ABC):
+    def __init__(
+            self,
+            model: Module,
+            mapping_rules_labels_t: np.ndarray,
+            model_input_x: TensorDataset,
+            rule_matches_z: np.ndarray,
+            dev_model_input_x: TensorDataset = None,
+            dev_gold_labels_y: TensorDataset = None,
+            trainer_config: BaseTrainerConfig = None,
+    ):
+        """
+        Constructor for each Trainer.
+            Args:
+                model: PyTorch model which will be used for final classification.
+                mapping_rules_labels_t: Mapping of rules to labels, binary encoded. Shape: rules x classes
+                model_input_x: Input tensors. These tensors will be fed to the provided model.
+                rule_matches_z: Binary encoded array of which rules matched. Shape: instances x rules
+                trainer_config: Config for different parameters like loss function, optimizer, batch size.
+        """
+        self.model = model
+        self.mapping_rules_labels_t = mapping_rules_labels_t
+        self.model_input_x = model_input_x
+        self.rule_matches_z = rule_matches_z
+        self.dev_model_input_x = dev_model_input_x
+        self.dev_gold_labels_y = dev_gold_labels_y
+
+        if trainer_config is None:
+            self.trainer_config = BaseTrainerConfig(model)
+        else:
+            self.trainer_config = trainer_config
+
+    @abstractmethod
+    def train(self, model_input_x: TensorDataset = None, rule_matches_z: np.ndarray = None):
+        pass
+
+    @abstractmethod
+    def test(self, test_features: TensorDataset, test_labels: TensorDataset):
+        pass
+
+    def initialise_optimizer(self):
+        try:
+            return self.trainer_config.optimizer(params=self.model.parameters(), lr=self.trainer_config.lr)
+        except TypeError:
+            logger.info("Wrong optimizer parameters. Optimizer should belong to torch.optim class or be PyTorch "
+                        "compatible.")
+
+
+class BaseTrainer(Trainer):
+
+    def _load_train_params(
+            self,
+            model_input_x: TensorDataset = None, rule_matches_z: np.ndarray = None,
+            dev_model_input_x: TensorDataset = None, dev_gold_labels_y: TensorDataset = None
+    ):
+        if model_input_x is not None and rule_matches_z is not None:
+            self.model_input_x = model_input_x
+            self.rule_matches_z = rule_matches_z
+        if dev_model_input_x is not None and dev_gold_labels_y is not None:
+            self.dev_model_input_x = dev_model_input_x
+            self.dev_gold_labels_y = dev_gold_labels_y
+
+    def _make_dataloader(
+            self, dataset: TensorDataset, shuffle: bool = True
+    ) -> DataLoader:
+        dataloader = DataLoader(
+            dataset,
+            batch_size=self.trainer_config.batch_size,
+            drop_last=False,
+            shuffle=shuffle,
+        )
+        return dataloader
+
+    def _load_batch(self, batch):
+
+        input_batch = [inp.to(self.trainer_config.device) for inp in batch[0: -1]]
+        label_batch = batch[-1].to(self.trainer_config.device)
+
+        return input_batch, label_batch
+
+    def _train_loop(
+            self, feature_label_dataloader, use_sample_weights: bool = False,
+            draw_plot: bool = False
+    ):
+        log_section("Training starts", logger)
+
+        self.model.to(self.trainer_config.device)
+        self.model.train()
+
+        train_losses, train_acc = [], []
+        if self.dev_model_input_x is not None:
+            dev_losses, dev_acc = [], []
+
+        for current_epoch in range(self.trainer_config.epochs):
+            logger.info("Epoch: {}".format(current_epoch))
+            epoch_loss, epoch_acc, steps = 0.0, 0.0, 0
+            for batch in tqdm(feature_label_dataloader):
+                input_batch, label_batch = self._load_batch(batch)
+                steps += 1
+
+                if use_sample_weights:
+                    input_batch, sample_weights = input_batch[:-1], input_batch[-1]
+
+                # forward pass
+                self.trainer_config.optimizer.zero_grad()
+                outputs = self.model(*input_batch)
+                if isinstance(outputs, torch.Tensor):
+                    logits = outputs
+                else:
+                    logits = outputs[0]
+
+                if use_sample_weights:
+                    loss_no_reduction = self.trainer_config.criterion(
+                        logits, label_batch, weight=self.trainer_config.class_weights, reduction="none"
+                    )
+                    loss = (loss_no_reduction * sample_weights).mean()
+                else:
+                    loss = self.trainer_config.criterion(logits, label_batch, weight=self.trainer_config.class_weights)
+
+                # backward pass
+                loss.backward()
+                if isinstance(self.trainer_config.grad_clipping, (int, float)):
+                    torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.trainer_config.grad_clipping)
+                self.trainer_config.optimizer.step()
+                acc = accuracy_of_probs(logits, label_batch)
+
+                epoch_loss += loss.detach().item()
+                epoch_acc += acc.item()
+
+                # print epoch loss and accuracy after each 10% of training is done
+                try:
+                    if steps % (int(round(len(feature_label_dataloader) / 10))) == 0:
+                        logger.info(f"Train loss: {epoch_loss / steps:.3f}, Train accuracy: {epoch_acc / steps:.3f}")
+                except ZeroDivisionError:
+                    continue
+
+            avg_loss = epoch_loss / len(feature_label_dataloader)
+            avg_acc = epoch_acc / len(feature_label_dataloader)
+            train_losses.append(avg_loss)
+            train_acc.append(avg_acc)
+
+            logger.info("Epoch train loss: {}".format(avg_loss))
+            logger.info("Epoch train accuracy: {}".format(avg_acc))
+
+            if self.dev_model_input_x is not None:
+                dev_clf_report, dev_loss = self.test(
+                    self.dev_model_input_x, self.dev_gold_labels_y, loss_calculation=True)
+                dev_losses.append(dev_loss)
+                dev_acc.append(dev_clf_report["accuracy"])
+                logger.info("Epoch development accuracy: {}".format(dev_clf_report["accuracy"]))
+
+            # saving model
+            if self.trainer_config.saved_models_dir is not None:
+                model_path = os.path.join(
+                    self.trainer_config.saved_models_dir,
+                    f"model_state_dict_epoch_{current_epoch}.pt"
+                )
+                torch.save(self.model.cpu().state_dict(), model_path)
+                self.model.to(self.trainer_config.device)
+
+        log_section("Training done", logger)
+
+        if draw_plot:
+            if self.dev_model_input_x:
+                draw_loss_accuracy_plot(
+                    {"train loss": train_losses, "train acc": train_acc, "dev loss": dev_losses, "dev acc": dev_acc}
+                )
+            else:
+                draw_loss_accuracy_plot({"train loss": train_losses, "train acc": train_acc})
+
+        self.model.eval()
+
+    def _prediction_loop(
+            self, feature_label_dataloader: DataLoader, loss_calculation: str = False
+    ) -> [np.ndarray, np.ndarray]:
+
+        self.model.to(self.trainer_config.device)
+        self.model.eval()
+        predictions_list, label_list = [], []
+        dev_loss, dev_acc = 0.0, 0.0
+
+        i = 0
+        # Loop over predictions
+        with torch.no_grad():
+            for batch in tqdm(feature_label_dataloader):
+                input_batch, label_batch = self._load_batch(batch)
+
+                # forward pass
+                self.trainer_config.optimizer.zero_grad()
+                outputs = self.model(*input_batch)
+                if isinstance(outputs, torch.Tensor):
+                    prediction_vals = outputs
+                else:
+                    prediction_vals = outputs[0]
+
+                if loss_calculation:
+                    dev_loss += self._calculate_dev_loss(prediction_vals, label_batch.long())
+
+                # add predictions and labels
+                predictions = np.argmax(prediction_vals.detach().cpu().numpy(), axis=-1)
+                predictions_list.append(predictions)
+                label_list.append(label_batch.detach().cpu().numpy())
+
+        predictions = np.squeeze(np.hstack(predictions_list))
+        gold_labels = np.squeeze(np.hstack(label_list))
+
+        return predictions, gold_labels, dev_loss
+
+    def _calculate_dev_loss(self, predictions: Tensor, labels: Tensor) -> Tensor:
+        """ Calculates the loss on the dev set using given criterion"""
+        predictions_one_hot = F.one_hot(predictions.argmax(1), num_classes=self.trainer_config.output_classes).float()
+        labels_one_hot = F.one_hot(labels, self.trainer_config.output_classes)
+        loss = self.trainer_config.criterion(predictions_one_hot, labels_one_hot)
+        return loss.detach()
+
+    def test(
+            self, features_dataset: TensorDataset, labels: TensorDataset, loss_calculation: bool = False
+    ) -> Tuple[Dict, Union[float, None]]:
+
+        feature_label_dataset = input_labels_to_tensordataset(features_dataset, labels.tensors[0].cpu().numpy())
+        feature_label_dataloader = self._make_dataloader(feature_label_dataset, shuffle=False)
+
+        predictions, gold_labels, dev_loss = self._prediction_loop(feature_label_dataloader, loss_calculation)
+
+        if self.trainer_config.evaluate_with_other_class:
+            clf_report = classification_report_other_class(
+                y_true=gold_labels, y_pred=predictions, ids2labels=self.trainer_config.ids2labels,
+                other_class_id=self.trainer_config.other_class_id
+            )
+        else:
+            clf_report = classification_report(y_true=gold_labels, y_pred=predictions, output_dict=True)
+
+        if loss_calculation:
+            return clf_report, dev_loss / len(feature_label_dataloader)
+        else:
+            return clf_report, None
diff --git a/knodle/trainer/utils/__init__.py b/knodle/trainer/utils/__init__.py
new file mode 100644
index 00000000..b14a87db
--- /dev/null
+++ b/knodle/trainer/utils/__init__.py
@@ -0,0 +1,1 @@
+from knodle.trainer.utils.utils import log_section
diff --git a/knodle/trainer/utils/checks.py b/knodle/trainer/utils/checks.py
new file mode 100644
index 00000000..d72cba26
--- /dev/null
+++ b/knodle/trainer/utils/checks.py
@@ -0,0 +1,15 @@
+import logging
+
+from numpy import ndarray
+from knodle.trainer.config import BaseTrainerConfig
+
+
+def check_other_class_id(trainer_config: BaseTrainerConfig, mapping_rules_labels_t: ndarray):
+    # check and derive other_class_id from class mappings if neccessary
+    if trainer_config.other_class_id is None:
+        if not trainer_config.filter_non_labelled:
+            trainer_config.other_class_id = mapping_rules_labels_t.shape[1]
+    elif trainer_config.other_class_id < 0:
+        raise RuntimeError("Label for negative samples should be greater than 0 for correct matrix multiplication")
+    elif trainer_config.other_class_id < mapping_rules_labels_t.shape[1] - 1:
+        logging.warning(f"Negative class {trainer_config.other_class_id} is already present in data")
diff --git a/knodle/trainer/utils/denoise.py b/knodle/trainer/utils/denoise.py
new file mode 100644
index 00000000..c0d127bc
--- /dev/null
+++ b/knodle/trainer/utils/denoise.py
@@ -0,0 +1,51 @@
+import logging
+from typing import Iterable
+from tqdm import tqdm
+
+import numpy as np
+import scipy.sparse as ss
+
+logger = logging.getLogger(__name__)
+
+
+def activate_neighbors(
+        rule_matches_z: np.ndarray, indices: Iterable[np.ndarray]
+) -> np.ndarray:
+    """
+    Take provided closest neighbors and add their rule matches.
+    If no indices provided for an instance, no activation is done (no matches added).
+    Args:
+        rule_matches_z: All rule matches. Shape: instances x rules
+        indices: Neighbor indices from knn per instance.
+    Returns:
+    """
+    if isinstance(rule_matches_z, ss.csr_matrix):
+        new_z_matrix = ss.lil_matrix(rule_matches_z.shape, dtype=np.int8)
+    else:
+        new_z_matrix = np.zeros_like(rule_matches_z)
+    
+    # make sure initial matches are preserved
+    new_z_matrix[rule_matches_z != 0] = 1
+
+    for index, neighbors in enumerate(tqdm(indices)):
+        if len(neighbors) == 0:
+            continue
+            
+        neighbors = neighbors.astype(np.int64)
+        neighborhood_z = rule_matches_z[neighbors, :].astype(np.int8)
+
+        activated_lfs = neighborhood_z.sum(axis=0)  # Add all lf activations
+
+        # All with != 0 are valid. We could also do some division here for weighted
+        activated_lfs = np.where(activated_lfs > 0)[0]
+
+        # add neighbor matches to the target object
+        new_z_matrix[index, activated_lfs] = 1
+        
+    # check out the amount of new matches
+    logger.info(f"Match increase: {new_z_matrix.sum() / rule_matches_z.sum() * 100 - 100:.2f}%")
+    
+    if isinstance(new_z_matrix, ss.lil_matrix):
+        return new_z_matrix.tocsr()
+    else:
+        return new_z_matrix
diff --git a/knodle/trainer/utils/utils.py b/knodle/trainer/utils/utils.py
new file mode 100644
index 00000000..a4d6a393
--- /dev/null
+++ b/knodle/trainer/utils/utils.py
@@ -0,0 +1,78 @@
+import random
+import logging
+
+import torch
+from torch import Tensor, argmax
+from torch.utils.data import TensorDataset
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+def log_section(text: str, logger: logging, additional_info: {} = None) -> None:
+    """
+    Prints a section
+    Args:
+        text: Text to print
+        logger: Logger object
+
+    Returns:
+
+    """
+    logger.info("======================================")
+    logger.info(text)
+    if additional_info:
+        for key, value in additional_info.items():
+            logger.info("{}: {}".format(key, value))
+    logger.info("======================================")
+
+
+def accuracy_of_probs(predictions: Tensor, ground_truth: Tensor):
+    """
+    Function to calculate the accuracy of two tensors with probabilities attached.
+    Args:
+        predictions: Predictions, shape: instances x labels
+        ground_truth: Ground truth, shape one of (instances x 1) or (instances x labels)
+
+    Returns: Accuracy
+
+    """
+    ground_truth = (
+        ground_truth if len(ground_truth.shape) == 1 else argmax(ground_truth, dim=-1)
+    )
+    y_pred_max = argmax(predictions, dim=-1)
+    correct_pred = (y_pred_max == ground_truth).float()
+    acc = correct_pred.sum() / len(correct_pred)
+    return acc
+
+
+def extract_tensor_from_dataset(dataset: TensorDataset, tensor_index: int) -> Tensor:
+    """
+    Extracts a tensor from a dataset.
+    Args:
+        dataset: Dataset to extract tensor from
+        tensor_index: Which tensor to extract
+
+    Returns: Tensor
+
+    """
+    return dataset.tensors[tensor_index]
+
+
+def check_and_return_device() -> torch.device:
+    """
+    Function to check if a GPU is available and sets the device for pytorch.
+    If a GPU is available -> The device is GPU, if not the device is CPU
+    """
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+    else:
+        device = torch.device("cpu")
+    return device
+
+
+def set_seed(seed: int) -> None:
+    """ Fix seed for all shuffle processes in order to get the reproducible result """
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
diff --git a/knodle/trainer/wscrossweigh/__init__.py b/knodle/trainer/wscrossweigh/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/trainer/wscrossweigh/config.py b/knodle/trainer/wscrossweigh/config.py
new file mode 100644
index 00000000..ac228133
--- /dev/null
+++ b/knodle/trainer/wscrossweigh/config.py
@@ -0,0 +1,61 @@
+from torch.optim import Optimizer
+
+from knodle.trainer.baseline.config import MajorityConfig
+from knodle.trainer.auto_config import AutoConfig
+
+
+@AutoConfig.register("wscrossweigh")
+class WSCrossWeighDenoisingConfig(MajorityConfig):
+    def __init__(
+            self,
+            partitions: int = 2,
+            folds: int = 10,
+            weight_reducing_rate: float = 0.5,
+            samples_start_weights: float = 2.0,
+            cw_epochs: int = None,
+            cw_batch_size: int = None,
+            cw_optimizer: Optimizer = None,
+            cw_lr: int = 0.1,
+            cw_filter_non_labelled: bool = None,
+            cw_other_class_id: int = None,
+            cw_grad_clipping: int = None,
+            cw_seed: int = None,
+            **kwargs
+    ):
+
+        super().__init__(**kwargs)
+        self.partitions = partitions
+        self.folds = folds
+        self.weight_reducing_rate = weight_reducing_rate
+        self.samples_start_weights = samples_start_weights
+
+        if cw_grad_clipping is None:
+            self.cw_grad_clipping = self.grad_clipping
+        else:
+            self.cw_grad_clipping = cw_grad_clipping
+
+        if cw_epochs is None:
+            self.cw_epochs = self.epochs
+        else:
+            self.cw_epochs = cw_epochs
+
+        if cw_batch_size is None:
+            self.cw_batch_size = self.batch_size
+        else:
+            self.cw_batch_size = cw_batch_size
+
+        if cw_optimizer is None:
+            self.cw_optimizer = self.optimizer
+        else:
+            self.cw_optimizer = cw_optimizer
+
+        if cw_filter_non_labelled is None and cw_other_class_id is None:
+            self.cw_filter_non_labelled = self.filter_non_labelled
+            self.cw_other_class_id = self.other_class_id
+        else:
+            self.cw_filter_non_labelled = cw_filter_non_labelled
+            self.cw_other_class_id = cw_other_class_id
+
+        self.cw_seed = cw_seed
+        self.cw_lr = cw_lr
+
diff --git a/knodle/trainer/wscrossweigh/utils.py b/knodle/trainer/wscrossweigh/utils.py
new file mode 100644
index 00000000..a9bf0ea5
--- /dev/null
+++ b/knodle/trainer/wscrossweigh/utils.py
@@ -0,0 +1,107 @@
+import logging
+import random
+from typing import Dict
+
+import numpy as np
+import torch
+from torch.utils.data import TensorDataset
+
+logger = logging.getLogger(__name__)
+
+
+def get_labels_randomly(
+        rule_matches_z: np.ndarray, rule_assignments_t: np.ndarray
+) -> np.ndarray:
+    """ Calculates sample labels basing on z and t matrices. If several patterns matched, select one randomly """
+
+    if rule_matches_z.shape[1] != rule_assignments_t.shape[0]:
+        raise ValueError("Dimensions mismatch!")
+
+    one_hot_labels = rule_matches_z.dot(rule_assignments_t)
+    one_hot_labels[one_hot_labels > 0] = 1
+    labels = [np.random.choice(np.where(r == 1)[0], 1)[0] for r in one_hot_labels]
+    return np.stack(labels, axis=0)
+
+
+def vocab_and_vectors(filename: str, special_tokens: list) -> (dict, dict, np.ndarray):
+    """special tokens have all-zero word vectors"""
+    with open(filename, encoding="UTF-8") as in_file:
+        parts = in_file.readline().strip().split(" ")
+        num_vecs = int(parts[0]) + len(special_tokens)  # + 1
+        dim = int(parts[1])
+
+        matrix = np.zeros((num_vecs, dim))
+        word_to_id = dict()
+
+        nextword_id = 0
+        for token in special_tokens:
+            word_to_id[token] = nextword_id
+            nextword_id += 1
+
+        for line in in_file:
+            parts = line.strip().split(" ")
+            word = parts[0]
+            if word not in word_to_id:
+                emb = [float(v) for v in parts[1:]]
+                matrix[nextword_id] = emb
+                word_to_id[word] = nextword_id
+                nextword_id += 1
+    return word_to_id, matrix
+
+
+def get_embedding_matrix(pretrained_embedding_file: str) -> np.ndarray:
+    """ Return matrix with pretrained glove embeddings"""
+    with open(pretrained_embedding_file, encoding="UTF-8") as in_file:
+        emb_matrix_size = in_file.readline().strip().split(" ")
+        embeddings = []
+        for line in in_file:
+            parts = line.strip().split(" ")
+            embeddings.append([float(v) for v in parts[1:]])
+        emb_matrix = np.array(embeddings)
+        assert emb_matrix.shape[0] == int(emb_matrix_size[0]) and emb_matrix.shape[
+            1
+        ] == int(emb_matrix_size[1])
+    return emb_matrix
+
+
+def set_device(enable_cuda: bool):
+    """ Set where the calculations will be done (cpu or cuda) depending on whether the cuda is available and chosen """
+    if enable_cuda and torch.cuda.is_available():
+        logger.info("Using GPU")
+        return torch.device("cuda")
+    else:
+        logger.info("Using CPU")
+        return torch.device("cpu")
+
+
+def check_splitting(
+        tst_samples: TensorDataset,
+        tst_labels: np.ndarray,
+        tst_idx: np.ndarray,
+        samples: torch.Tensor,
+        labels: np.ndarray,
+) -> None:
+    """ Custom function to check that the splitting into train and test sets fro WSCrossWeigh was done correctly"""
+
+    rnd_tst = np.random.randint(0, tst_samples.tensors[0].shape[0])  # take some random index
+    tst_sample = tst_samples.tensors[0][rnd_tst, :]
+    tst_idx = tst_idx[rnd_tst]
+    tst_label = tst_labels[rnd_tst, :]
+
+    if not torch.equal(tst_sample, samples[tst_idx, :]):
+        raise RuntimeError(
+            "The splitting of original training set into cw train and test sets have been done "
+            "incorrectly! A sample does not correspond to one in original dataset"
+        )
+
+    if not np.array_equal(tst_label, labels[tst_idx, :]):
+        raise RuntimeError(
+            "The splitting of original training set into cw train and test sets have been done "
+            "incorrectly! A sample label does not correspond to one in original dataset"
+        )
+
+
+def return_unique(where_to_find: np.ndarray, what_to_find: np.ndarray) -> np.ndarray:
+    """ Checks intersections between the 1st and the 2nd arrays and return unique values of the 1st array """
+    intersections = np.intersect1d(where_to_find, what_to_find, return_indices=True)[1].tolist()
+    return np.delete(where_to_find, intersections)
diff --git a/knodle/trainer/wscrossweigh/wscrossweigh.py b/knodle/trainer/wscrossweigh/wscrossweigh.py
new file mode 100644
index 00000000..3f096df3
--- /dev/null
+++ b/knodle/trainer/wscrossweigh/wscrossweigh.py
@@ -0,0 +1,135 @@
+import logging
+import os
+from copy import copy
+
+import numpy as np
+import torch
+from joblib import load
+from torch.nn import Module
+from torch.optim import SGD
+from torch.utils.data import TensorDataset
+
+from knodle.trainer.baseline.majority import MajorityVoteTrainer
+from knodle.trainer.wscrossweigh.config import WSCrossWeighDenoisingConfig
+from knodle.trainer.wscrossweigh.wscrossweigh_weights_calculator import WSCrossWeighWeightsCalculator
+
+from knodle.transformation.filter import filter_empty_probabilities
+from knodle.transformation.majority import z_t_matrices_to_majority_vote_probs
+from knodle.transformation.torch_input import input_info_labels_to_tensordataset
+
+torch.set_printoptions(edgeitems=100)
+logger = logging.getLogger(__name__)
+logging.getLogger('matplotlib.font_manager').disabled = True
+
+
+class WSCrossWeighTrainer(MajorityVoteTrainer):
+
+    def __init__(
+            self,
+            cw_model: Module = None,
+            cw_model_input_x: TensorDataset = None,
+            cw_rule_matches_z: np.ndarray = None,
+            run_classifier: bool = True,  # set to False if you want only the calculation of the sample weights
+            use_weights: bool = True,  # set to False if you want to use weights = 1 (baseline)
+            **kwargs
+    ):
+        self.cw_model = cw_model if cw_model else kwargs.get("model")
+        self.cw_model_input_x = cw_model_input_x if cw_model_input_x else kwargs.get("model_input_x")
+        self.cw_rule_matches_z = cw_rule_matches_z if cw_rule_matches_z else kwargs.get("rule_matches_z")
+
+        if kwargs.get("trainer_config") is None:
+            kwargs["trainer_config"] = WSCrossWeighDenoisingConfig(
+                optimizer=SGD,
+                cw_optimizer=SGD,
+                lr=0.001,
+                cw_lr=0.001,
+            )
+        super().__init__(**kwargs)
+
+        self.run_classifier = run_classifier
+        self.use_weights = use_weights
+
+        logger.info("CrossWeigh Config is used: {}".format(self.trainer_config.__dict__))
+
+    def train(
+            self,
+            model_input_x: TensorDataset = None, rule_matches_z: np.ndarray = None,
+            dev_model_input_x: TensorDataset = None, dev_gold_labels_y: TensorDataset = None
+    ):
+        """ This function sample_weights the samples with WSCrossWeigh method and train the model """
+        self._load_train_params(model_input_x, rule_matches_z, dev_model_input_x, dev_gold_labels_y)
+
+        # initialise optimizer
+        self.trainer_config.optimizer = self.initialise_optimizer()
+
+        train_labels = self.calculate_labels()
+
+        sample_weights = self._get_sample_weights() if self.use_weights \
+            else torch.FloatTensor([1] * len(self.model_input_x))
+
+        if not self.run_classifier:
+            logger.info("No classifier is to be trained")
+            return
+        logger.info("Classifier training is started")
+
+        train_loader = self._make_dataloader(
+            input_info_labels_to_tensordataset(self.model_input_x, sample_weights.cpu().numpy(), train_labels)
+        )
+
+        self._train_loop(train_loader, use_sample_weights=True, draw_plot=True)
+
+    def calculate_labels(self) -> np.ndarray:
+        """ This function calculates label probabilities and filter out non labelled samples, when needed """
+        train_labels = z_t_matrices_to_majority_vote_probs(
+            self.rule_matches_z, self.mapping_rules_labels_t, self.trainer_config.other_class_id
+        )
+
+        if self.trainer_config.filter_non_labelled:
+            self.model_input_x, train_labels, self.rule_matches_z = filter_empty_probabilities(
+                self.model_input_x, train_labels, self.rule_matches_z
+            )
+
+        if train_labels.shape[1] != self.trainer_config.output_classes:
+            raise ValueError(
+                f"The number of output classes {self.trainer_config.output_classes} do not correspond to labels "
+                f"probabilities dimension {train_labels.shape[1]}"
+            )
+
+        return train_labels
+
+    def _get_sample_weights(self) -> torch.FloatTensor:
+        """ This function checks whether there are accessible already pretrained sample weights. If yes, return
+        them. If not, calculates sample weights calling method of WSCrossWeighWeightsCalculator class"""
+
+        if os.path.isfile(os.path.join(
+                self.trainer_config.caching_folder, f"sample_weights_{self.trainer_config.caching_suffix}.lib")
+        ):
+            logger.info("Already pretrained samples sample_weights will be used.")
+            sample_weights = load(os.path.join(
+                self.trainer_config.caching_folder, f"sample_weights_{self.trainer_config.caching_suffix}.lib")
+            )
+        else:
+            logger.info("No pretrained sample weights are found, they will be calculated now")
+            sample_weights = WSCrossWeighWeightsCalculator(
+                model=self.cw_model,
+                mapping_rules_labels_t=self.mapping_rules_labels_t,
+                model_input_x=self.cw_model_input_x,
+                rule_matches_z=self.cw_rule_matches_z,
+                trainer_config=self.get_denoising_config(),
+            ).calculate_weights()
+            logger.info(f"Sample weights are calculated and saved to {self.trainer_config.caching_folder} folder")
+        return sample_weights
+
+    def get_denoising_config(self):
+        """ Get a config for WSCrossWeigh sample weights calculation """
+        weights_calculation_config = copy(self.trainer_config)
+        weights_calculation_config.epochs = self.trainer_config.cw_epochs
+        weights_calculation_config.optimizer = self.trainer_config.cw_optimizer
+        weights_calculation_config.lr = self.trainer_config.cw_lr
+        weights_calculation_config.batch_size = self.trainer_config.cw_batch_size
+        weights_calculation_config.filter_non_labelled = self.trainer_config.cw_filter_non_labelled
+        weights_calculation_config.other_class_id = self.trainer_config.cw_other_class_id
+        weights_calculation_config.grad_clipping = self.trainer_config.cw_grad_clipping
+        weights_calculation_config.seed = self.trainer_config.cw_seed
+        weights_calculation_config.saved_models_dir = None
+        return weights_calculation_config
diff --git a/knodle/trainer/wscrossweigh/wscrossweigh_weights_calculator.py b/knodle/trainer/wscrossweigh/wscrossweigh_weights_calculator.py
new file mode 100644
index 00000000..749d9151
--- /dev/null
+++ b/knodle/trainer/wscrossweigh/wscrossweigh_weights_calculator.py
@@ -0,0 +1,222 @@
+import copy
+import logging
+import os
+import random
+from typing import Dict, List
+
+import numpy as np
+import scipy.sparse as sp
+import torch
+from torch.utils.data import TensorDataset, DataLoader
+from joblib import dump
+
+from knodle.trainer.baseline.majority import MajorityVoteTrainer
+from knodle.trainer.wscrossweigh.utils import check_splitting, return_unique
+from knodle.trainer.utils import log_section
+from knodle.transformation.filter import filter_empty_probabilities
+from knodle.transformation.majority import z_t_matrices_to_majority_vote_probs
+from knodle.transformation.torch_input import input_info_labels_to_tensordataset, input_labels_to_tensordataset
+
+logger = logging.getLogger(__name__)
+torch.set_printoptions(edgeitems=100)
+
+
+class WSCrossWeighWeightsCalculator(MajorityVoteTrainer):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.wscrossweigh_model = copy.deepcopy(self.model)
+        self.sample_weights = torch.empty(0)
+
+    def calculate_weights(self) -> torch.FloatTensor:
+        """
+        This function calculates the sample_weights for samples using WSCrossWeigh method
+        :return matrix of the sample sample_weights
+        """
+
+        if self.trainer_config.folds < 2:
+            raise ValueError("Number of folds should be at least 2 to perform WSCrossWeigh denoising")
+
+        logger.info("======= Denoising with WSCrossWeigh is started =======")
+        os.makedirs(self.trainer_config.caching_folder, exist_ok=True)
+
+        labels = z_t_matrices_to_majority_vote_probs(
+            self.rule_matches_z, self.mapping_rules_labels_t, self.trainer_config.other_class_id
+        )
+
+        if self.trainer_config.filter_non_labelled:
+            self.model_input_x, labels, self.rule_matches_z = filter_empty_probabilities(
+                self.model_input_x, labels, self.rule_matches_z
+            )
+
+        other_sample_ids = self._get_other_sample_ids(labels) if self.trainer_config.other_class_id else None
+        rules_samples_ids_dict = self._get_rules_samples_ids_dict()
+
+        self.sample_weights = self.initialise_sample_weights()
+
+        for partition in range(self.trainer_config.partitions):
+            log_section(f"CrossWeigh Partition {partition + 1}/{self.trainer_config.partitions}:", logger)
+
+            shuffled_rules_ids = self._get_shuffled_rules_idx()  # shuffle anew for each cw round
+
+            for fold in range(self.trainer_config.folds):
+                # for each fold the model is trained from scratch
+                self.wscrossweigh_model = copy.deepcopy(self.model).to(self.trainer_config.device)
+                train_loader, test_loader = self.get_cw_data(
+                    shuffled_rules_ids, rules_samples_ids_dict, labels, fold, other_sample_ids
+                )
+                self._train_loop(train_loader)
+                self.cw_test(test_loader)
+
+            log_section(f"CrossWeigh Partition {partition + 1} is done", logger)
+
+        dump(
+            self.sample_weights, os.path.join(self.trainer_config.caching_folder,
+                                              f"sample_weights_{self.trainer_config.caching_suffix}.lib")
+             )
+        logger.info("======= Denoising with WSCrossWeigh is completed =======")
+        return self.sample_weights
+
+    def _get_other_sample_ids(self, labels: np.ndarray) -> List[int]:
+        return np.where(labels[:, self.trainer_config.other_class_id] == 1)[0].tolist()
+
+    def _get_shuffled_rules_idx(self) -> List[int]:
+        """ Get shuffled row indices of dataset """
+        rel_rules_ids = [rule_idx for rule_idx in range(0, self.mapping_rules_labels_t.shape[0])]
+        random.shuffle(rel_rules_ids)
+        return rel_rules_ids
+
+    def _get_rules_samples_ids_dict(self):
+        """
+        This function creates a dictionary {rule id : sample id where this rule matched}. The dictionary is needed as a
+        support tool for faster calculation of cw train and cw test sets
+         """
+        if isinstance(self.rule_matches_z, sp.csr_matrix):
+            rules_samples_ids_dict = {key: [] for key in range(self.rule_matches_z.shape[1])}
+            for row, col in zip(*self.rule_matches_z.nonzero()):
+                rules_samples_ids_dict[col].append(row)
+        else:
+            rules_samples_ids_dict = dict((i, set()) for i in range(0, self.rule_matches_z.shape[1]))
+            for row_idx, row in enumerate(self.rule_matches_z):
+                rules = np.where(row == 1)[0].tolist()
+                for rule in rules:
+                    rules_samples_ids_dict[rule].add(row_idx)
+        return rules_samples_ids_dict
+
+    def initialise_sample_weights(self) -> torch.FloatTensor:
+        """ Initialise a sample_weights matrix (num_samples x 1): weights for all samples equal sample_start_weights """
+        return torch.FloatTensor([self.trainer_config.samples_start_weights] * self.model_input_x.tensors[0].shape[0])
+
+    def calculate_rules_indices(self, rules_idx: list, fold: int) -> (np.ndarray, np.ndarray):
+        """
+        Calculates the indices of the samples which are to be included in WSCrossWeigh training and test sets
+        :param rules_idx: all rules indices (shuffled) that are to be splitted into cw training & cw test set rules
+        :param fold: number of a current hold-out fold
+        :return: two arrays containing indices of rules that will be used for cw training and cw test set accordingly
+        """
+        test_rules_idx = rules_idx[fold::self.trainer_config.folds]
+        train_rules_idx = [rules_idx[x::self.trainer_config.folds] for x in range(self.trainer_config.folds)
+                           if x != fold]
+        train_rules_idx = [ids for sublist in train_rules_idx for ids in sublist]
+
+        if not set(test_rules_idx).isdisjoint(set(train_rules_idx)):
+            raise ValueError("Splitting into train and test rules is done incorrectly.")
+
+        return train_rules_idx, test_rules_idx
+
+    def get_cw_data(
+            self, rules_ids: List[int], rules_samples_ids_dict: Dict, labels: np.ndarray, fold: int,
+            other_sample_ids: List[int]
+    ) -> (DataLoader, DataLoader):
+        """
+        This function returns train and test dataloaders for WSCrossWeigh training. Each dataloader comprises encoded
+        samples, labels and sample indices in the original matrices
+        :param rules_ids: shuffled rules indices
+        :param labels: labels of all training samples
+        :param fold: number of a current hold-out fold
+        :return: dataloaders for cw training and testing
+        """
+        train_rules_idx, test_rules_idx = self.calculate_rules_indices(rules_ids, fold)
+
+        # select train and test samples and labels according to the selected rules idx
+        test_samples, test_labels, test_idx = self._get_cw_samples_labels_idx(
+            labels, test_rules_idx, rules_samples_ids_dict
+        )
+        test_loader = self._make_dataloader(
+            input_info_labels_to_tensordataset(test_samples, test_idx, test_labels)
+        )
+
+        train_samples, train_labels, _ = self._get_cw_samples_labels_idx(
+            labels, train_rules_idx, rules_samples_ids_dict, test_idx, other_sample_ids
+        )
+        train_loader = self._make_dataloader(
+            input_labels_to_tensordataset(train_samples, train_labels)
+        )
+
+        logger.info(f"Fold {fold}/{self.trainer_config.folds}  Rules in training set: {len(train_rules_idx)}, "
+                    f"rules in test set: {len(test_rules_idx)}, samples in training set: {len(train_samples)}, "
+                    f"samples in test set: {len(test_samples)}")
+        return train_loader, test_loader
+
+    def _get_cw_samples_labels_idx(
+            self, labels: np.ndarray, indices: list, rules_samples_ids_dict: Dict, check_intersections: np.ndarray = None,
+            other_sample_ids: list = None
+    ) -> (torch.Tensor, np.ndarray, np.ndarray):
+        """
+        Extracts the samples and labels from the original matrices by indices. If intersection is filled with
+        another sample matrix, it also checks whether the sample is not in this other matrix yet.
+        :param labels: all training samples labels, shape=(num_samples, num_classes)
+        :param indices: indices of rules; samples, where these rules matched & their labels are to be included in set
+        :param rules_samples_ids_dict: dictionary {rule_id : sample_ids}
+        :param check_intersections: optional parameter that indicates that intersections should be checked (used to
+        exclude the sentences from the WSCrossWeigh training set which are already in WSCrossWeigh test set)
+        :return: samples, labels and indices in the original matrix
+        """
+        sample_ids = [list(rules_samples_ids_dict.get(idx)) for idx in indices]
+        sample_ids = list(set([value for sublist in sample_ids for value in sublist]))
+
+        if other_sample_ids is not None:
+            sample_ids = list(set(sample_ids).union(set(other_sample_ids)))
+
+        if check_intersections is not None:
+            sample_ids = return_unique(np.array(sample_ids), check_intersections)
+        cw_samples_dataset = TensorDataset(torch.Tensor(self.model_input_x.tensors[0][sample_ids]))
+        cw_labels = np.array(labels[sample_ids])
+        cw_samples_idx = np.array(sample_ids)
+
+        check_splitting(cw_samples_dataset, cw_labels, cw_samples_idx, self.model_input_x.tensors[0], labels)
+        return cw_samples_dataset, cw_labels, cw_samples_idx
+
+    def cw_test(self, test_loader: DataLoader) -> None:
+        """
+        This function tests of trained WSCrossWeigh model on a hold-out fold, compared the predicted labels with the
+        ones got with weak supervision and reduces sample_weights of disagreed samples
+        :param test_loader: loader with the data which is used for testing (hold-out fold)
+        """
+        self.wscrossweigh_model.eval()
+        correct_predictions, wrong_predictions = 0, 0
+
+        with torch.no_grad():
+            for batch in test_loader:
+                features, labels = self._load_batch(batch)
+                data_features, data_indices = features[:-1], features[-1]
+
+                outputs = self.wscrossweigh_model(*data_features)
+                outputs = outputs[0] if not isinstance(outputs, torch.Tensor) else outputs
+                _, predicted = torch.max(outputs.data, -1)
+                predictions = predicted.tolist()
+
+                for curr_pred in range(len(predictions)):
+                    gold = labels.tolist()[curr_pred]
+                    gold_classes = gold.index(max(gold))
+                    guess = predictions[curr_pred]
+                    if guess != gold_classes:       # todo: what if more than one class could be predicted? e.g. conll
+                        wrong_predictions += 1
+                        curr_id = data_indices[curr_pred].tolist()
+                        self.sample_weights[curr_id] *= self.trainer_config.weight_reducing_rate
+                    else:
+                        correct_predictions += 1
+
+        logger.info("Correct predictions: {:.3f}%, wrong predictions: {:.3f}%".format(
+            correct_predictions * 100 / (correct_predictions + wrong_predictions),
+            wrong_predictions * 100 / (correct_predictions + wrong_predictions)))
diff --git a/knodle/transformation/__init__.py b/knodle/transformation/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/transformation/filter.py b/knodle/transformation/filter.py
new file mode 100644
index 00000000..c2400ba0
--- /dev/null
+++ b/knodle/transformation/filter.py
@@ -0,0 +1,51 @@
+from typing import Tuple, Union
+
+import numpy as np
+from torch.utils.data import TensorDataset
+
+
+def filter_empty_probabilities(
+        input_data_x: TensorDataset, class_probas_y: np.ndarray, rule_matches_z: np.ndarray = None
+) -> Union[Tuple[TensorDataset, np.ndarray, np.ndarray], Tuple[TensorDataset, np.ndarray]]:
+    """Delete rows of TensorDataset's where the cumulative probability equals 0.
+    Args:
+        input_data_x: A TensorDataset serving as input to a model
+        class_probas_y: Array, holding class probabilities, shape=num_samples, num_classes
+        rule_matches_z: optional array with rules matched in samples. If given, will also be filtered
+    :return: Modified TensorDataset's
+    """
+    if len(class_probas_y.shape) != 2:
+        raise ValueError("y_probs needs to be a matrix of dimensions num_samples x num_classes")
+
+    prob_sums = class_probas_y.sum(axis=-1)
+    non_zeros = np.where(prob_sums != 0)[0]
+
+    new_tensors = []
+    for i in range(len(input_data_x.tensors)):
+        new_tensors.append(input_data_x.tensors[i][non_zeros])
+    new_x = TensorDataset(*new_tensors)
+
+    if rule_matches_z is not None:
+        return new_x, class_probas_y[non_zeros], rule_matches_z[non_zeros]
+
+    return new_x, class_probas_y[non_zeros]
+
+
+def filter_unconclusive_probabilities(
+        input_data_x: TensorDataset, class_probas_y: np.ndarray, rule_matches_z: np.ndarray = None,
+        probability_threshold: float = 0.7
+) -> Union[Tuple[TensorDataset, np.ndarray, np.ndarray], Tuple[TensorDataset, np.ndarray]]:
+    """Filters instances where no single class probability exceeds "probability_threshold".
+    """
+    prob_sums = class_probas_y.max(axis=-1)
+    conclusive_idx = np.where(prob_sums >= probability_threshold)[0]
+
+    new_tensors = []
+    for i in range(len(input_data_x.tensors)):
+        new_tensors.append(input_data_x.tensors[i][conclusive_idx])
+    new_x = TensorDataset(*new_tensors)
+
+    if rule_matches_z is not None:
+        return new_x, class_probas_y[conclusive_idx], rule_matches_z[conclusive_idx]
+
+    return new_x, class_probas_y[conclusive_idx]
diff --git a/knodle/transformation/labels.py b/knodle/transformation/labels.py
new file mode 100644
index 00000000..cbf7c9d7
--- /dev/null
+++ b/knodle/transformation/labels.py
@@ -0,0 +1,20 @@
+from typing import Dict
+import numpy as np
+
+def translate_predictions(predictions: np.ndarray, labels: np.ndarray, ids2labels: Dict) -> Dict:
+    """
+    Prepare data for string label based evaluation.
+    :param predictions: predicted label ids
+    :param labels: gold label ids
+    :param ids2labels: translation dictionary from string labels to label ids
+    :return:
+    """
+    predictions_idx = predictions.astype(int).tolist()
+    labels_idx = labels.astype(int).tolist()
+
+    predictions = [ids2labels[p] for p in predictions_idx]
+    test_labels = [ids2labels[p] for p in labels_idx]
+    return predictions, test_labels
+
+
+# TODO: add function adding "other" class, if no label is given.
\ No newline at end of file
diff --git a/knodle/transformation/majority.py b/knodle/transformation/majority.py
new file mode 100644
index 00000000..d190f8ac
--- /dev/null
+++ b/knodle/transformation/majority.py
@@ -0,0 +1,115 @@
+import numpy as np
+import scipy.sparse as sp
+from torch.utils.data import TensorDataset
+import warnings
+from knodle.transformation.filter import filter_empty_probabilities
+
+
+def probabilies_to_majority_vote(
+        probs: np.ndarray, choose_random_label: bool = True, other_class_id: int = None
+) -> int:
+    """Transforms a vector of probabilities to its majority vote. If there is one class with clear majority, return it.
+    If there are more than one class with equal probabilities: either select one of the classes randomly or assign to
+    the sample the other class id.
+ 
+    Args:
+        probs: Vector of probabilities for 1 sample. Shape: classes x 1
+        choose_random_label: Choose a random label, if there's no clear majority.
+        other_class_id: Class ID being used, if there's no clear majority
+    Returns: An array of classes.
+    """
+    if choose_random_label and other_class_id is not None:
+        raise ValueError("You can either choose a random class, or transform undefined cases to an other class.")
+
+    row_max = np.max(probs)
+    num_occurrences = (row_max == probs).sum()
+    if num_occurrences == 1:
+        return int(np.argmax(probs))
+    elif choose_random_label:
+        max_ids = np.where(probs == row_max)[0]
+        return int(np.random.choice(max_ids))
+    elif other_class_id is not None:
+        return other_class_id
+    else:
+        raise ValueError("Specify a way how to resolve unclear majority votes.")
+
+
+def z_t_matrices_to_majority_vote_probs(
+        rule_matches_z: np.ndarray, mapping_rules_labels_t: np.ndarray, other_class: int = None
+) -> np.ndarray:
+    """
+    This function calculates a majority vote probability for all rule_matches_z. The difference from simple
+    get_majority_vote_probs function is the following: samples, where no rules matched (that is, all elements in
+    the corresponding raw in rule_matches_z matrix equal 0), are assigned to no_match_class (that is, a value in the
+    corresponding column in rule_counts_probs matrix is changed to 1).
+
+    Args:
+        rule_matches_z: Binary encoded array of which rules matched. Shape: instances x rules
+        mapping_rules_labels_t: Mapping of rules to labels, binary encoded. Shape: rules x classes
+        other_class: Class which is chosen, if no function is hitting.
+    Returns: Array with majority vote probabilities. Shape: instances x classes
+    """
+
+    if rule_matches_z.shape[1] != mapping_rules_labels_t.shape[0]:
+        raise ValueError(f"Dimensions mismatch! Z matrix has shape {rule_matches_z.shape}, while "
+                         f"T matrix has shape {mapping_rules_labels_t.shape}")
+
+    if isinstance(rule_matches_z, sp.csr_matrix):
+        rule_counts = rule_matches_z.dot(mapping_rules_labels_t)
+        if isinstance(rule_counts, sp.csr_matrix):
+            rule_counts = rule_counts.toarray()
+    else:
+        rule_counts = np.matmul(rule_matches_z, mapping_rules_labels_t)
+
+    if other_class:
+        if other_class < 0:
+            raise RuntimeError("Label for negative samples should be greater than 0 for correct matrix multiplication")
+        if other_class < mapping_rules_labels_t.shape[1] - 1:
+            warnings.warn(f"Negative class {other_class} is already present in data")
+        if rule_counts.shape[1] == other_class:
+            rule_counts = np.hstack((rule_counts, np.zeros([rule_counts.shape[0], 1])))
+            rule_counts[~rule_counts.any(axis=1), other_class] = 1
+        elif rule_counts.shape[1] >= other_class:
+            rule_counts[~rule_counts.any(axis=1), other_class] = 1
+        else:
+            raise ValueError("Other class id is incorrect")
+    rule_counts_probs = rule_counts / rule_counts.sum(axis=1).reshape(-1, 1)
+    rule_counts_probs[np.isnan(rule_counts_probs)] = 0
+    return rule_counts_probs
+
+
+def z_t_matrices_to_majority_vote_labels(
+        rule_matches_z: np.ndarray, mapping_rules_labels_t: np.ndarray,
+        choose_random_label: bool = True, other_class_id: int = None
+) -> np.array:
+    """Computes the majority labels. If no clear "winner" is found, no_rule_label is used instead.
+    Args:
+        rule_matches_z: Binary encoded array of which rules matched. Shape: instances x rules
+        mapping_rules_labels_t: Mapping of rules to labels, binary encoded. Shape: rules x classes
+        choose_random_label: Whether a random label is chosen, if there's no clear majority vote.
+        other_class_id: Dummy label
+    Returns: Decision per sample. Shape: (instances, )
+    """
+    rule_counts_probs = z_t_matrices_to_majority_vote_probs(rule_matches_z, mapping_rules_labels_t)
+
+    kwargs = {"choose_random_label": choose_random_label, "other_class_id": other_class_id}
+    majority_labels = np.apply_along_axis(probabilies_to_majority_vote, axis=1, arr=rule_counts_probs, **kwargs)
+    return majority_labels
+
+
+def input_to_majority_vote_input(
+        input_data_x: TensorDataset, rule_matches_z: np.ndarray, mapping_rules_labels_t: np.ndarray,
+        filter_non_labelled: bool = True, other_class_id: int = None, use_probabilistic_labels: bool = True
+):
+    if other_class_id is not None and filter_non_labelled:
+        raise ValueError("You can either filter samples with no weak labels or add them to 'other_class_id'")
+
+    label_probs = z_t_matrices_to_majority_vote_probs(rule_matches_z, mapping_rules_labels_t, other_class_id)
+    if filter_non_labelled:
+        input_data_x, label_probs = filter_empty_probabilities(input_data_x, label_probs)
+
+    if not use_probabilistic_labels:
+        kwargs = {"choose_random_label": True, "other_class_id": other_class_id}
+        label_probs = np.apply_along_axis(probabilies_to_majority_vote, axis=1, arr=label_probs, **kwargs)
+
+    return input_data_x, label_probs
diff --git a/knodle/transformation/rule_label_format.py b/knodle/transformation/rule_label_format.py
new file mode 100644
index 00000000..95a1d9c0
--- /dev/null
+++ b/knodle/transformation/rule_label_format.py
@@ -0,0 +1,30 @@
+import numpy as np
+
+
+def transform_snorkel_matrix_to_z_t(class_matrix: np.ndarray) -> [np.ndarray, np.ndarray]:
+    """Takes a matrix in format used by e.g. Snorkel (https://github.com/snorkel-team/snorkel)
+    and transforms it to z / t matrices. Format
+        - class_matrix_ij = -1, iff the rule doesn't apply
+        - class_matrix_ij = k, iff the rule labels class k
+
+    :param class_matrix: shape=(num_samples, num_weak_labellers)
+    :return: Z, T matrix as described in the paper.
+    """
+    num_classes = class_matrix.max() + 1
+
+    z_matrix = np.where(class_matrix != -1, 1, class_matrix)
+    z_matrix = np.where(z_matrix == -1, 0, z_matrix)
+
+    t_matrix = np.zeros((class_matrix.shape[1], num_classes))
+
+    for i in range(class_matrix.shape[1]):
+        row = class_matrix[:, i].tolist()
+        label = max(row)
+        if set(row) != {-1, label}:
+            raise RuntimeError(
+                "A weak labeller is not allowed to label more than one class."
+            )
+
+        t_matrix[i, label] = 1
+
+    return z_matrix, t_matrix
diff --git a/knodle/transformation/torch_input.py b/knodle/transformation/torch_input.py
new file mode 100644
index 00000000..349348bf
--- /dev/null
+++ b/knodle/transformation/torch_input.py
@@ -0,0 +1,28 @@
+import numpy as np
+import torch
+from torch.utils.data import TensorDataset
+
+
+def input_labels_to_tensordataset(model_input_x: TensorDataset, labels: np.ndarray) -> TensorDataset:
+    """
+    This function takes Dataset with data features (num_samples x features dimension x features) and
+    labels (num_samples x labels dimension) and turns it into one Dataset
+    """
+    model_tensors = model_input_x.tensors
+    input_label_dataset = TensorDataset(*model_tensors, torch.from_numpy(labels))
+
+    return input_label_dataset
+
+
+def input_info_labels_to_tensordataset(
+        model_input_x: TensorDataset, input_info: np.ndarray, labels: np.ndarray
+) -> TensorDataset:
+    """
+    This function takes Dataset with data features (num_samples x features dimension x features),
+    labels (num_samples x labels dimension) and some additional information about data encoded as
+    a numpy array (num_samples x n; could be sample weights, sample indices etc) and turns it into one Dataset
+    """
+    model_tensors = model_input_x.tensors
+    input_ids_label_dataset = TensorDataset(*model_tensors, torch.from_numpy(input_info), torch.from_numpy(labels))
+
+    return input_ids_label_dataset
diff --git a/knodle/utils/__init__.py b/knodle/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/knodle/version.py b/knodle/version.py
new file mode 100644
index 00000000..058ef7a7
--- /dev/null
+++ b/knodle/version.py
@@ -0,0 +1,3 @@
+# This is a placeholder file and is not supposed to be maintained.
+# This file will be replaced with an up-to-date value during package build via github action.
+__version__ = '0.0.0'
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..374b58cb
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000..800d32a4
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,19 @@
+# General
+joblib
+tqdm
+
+# Computation
+numpy<1.20.0,>=1.16.5
+scipy
+pandas
+
+# ML frameworks
+scikit-learn
+torch
+snorkel<=0.9.7
+
+# kNN
+annoy
+
+# graphics
+matplotlib
diff --git a/requirements_dev.txt b/requirements_dev.txt
deleted file mode 100644
index c7444272..00000000
--- a/requirements_dev.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-flake8
-pytest
-pytest-cov
--e .
-wheel
\ No newline at end of file
diff --git a/setup.py b/setup.py
index e62cdc8d..0b0eed33 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,7 @@
 import io
 import os
 import re
+from typing import Dict
 
 from setuptools import find_packages
 from setuptools import setup
@@ -9,36 +10,47 @@
 def read(filename):
     filename = os.path.join(os.path.dirname(__file__), filename)
     text_type = type(u"")
-    with io.open(filename, mode="r", encoding='utf-8') as fd:
-        return re.sub(text_type(r':[a-z]+:`~?(.*?)`'), text_type(r'``\1``'), fd.read())
+    with io.open(filename, mode="r", encoding="utf-8") as fd:
+        return re.sub(text_type(r":[a-z]+:`~?(.*?)`"), text_type(r"``\1``"), fd.read())
 
 
+VERSION: Dict[str, str] = {}
+with open("knodle/version.py", "r") as version_file:
+    exec(version_file.read(), VERSION)
+
+with open("requirements.txt") as f:
+    requirements = f.readlines()
+
+test_requirements = ["pytest", "pytest-cov"]
+
+# use external package to extract tag of the latest release as version
+# https://pypi.org/project/setuptools-git-versioning/
+
 setup(
     name="knodle",
-    version="0.1.0",
-    url="https://github.com/knodle/knodle",
-    license='TBC',
-
+    version=VERSION.get("__version__"),
+    url="http://knodle.cc",
+    project_urls={
+        "github": "https://github.com/knodle/knodle",
+        "Bug Tracker": "https://github.com/knodle/knodle/issues"
+    },
+    license="Apache 2.0",
     author="knodle",
-    author_email="alessandro.volpicella@gmail.com",
-
+    author_email="knodle@cs.univie.ac.at",
     description="Knowledge infused deep learning framework",
     long_description=read("README.md"),
-
-    packages=find_packages(exclude=('tests',)),
-
-    install_requires=[],
-
+    long_description_content_type="text/markdown",
+    packages=find_packages(exclude=["tests"]),
+    package_dir={'knodle': 'knodle'},
+    install_requires=requirements,
+    tests_require=test_requirements,
+    extras_require={"test": test_requirements},
     classifiers=[
-        'Development Status :: 2 - Pre-Alpha',
-        'License :: OSI Approved :: MIT License',
-        'Programming Language :: Python',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
+        "Development Status :: 2 - Pre-Alpha",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8"
     ],
 )
diff --git a/update_version_from_tag.py b/update_version_from_tag.py
new file mode 100644
index 00000000..7aefa011
--- /dev/null
+++ b/update_version_from_tag.py
@@ -0,0 +1,12 @@
+import sys
+import re
+
+tag = sys.argv[1]
+version_pattern = re.compile("\d+(.\d+)+(.(dev|post)\d+)?")
+version = re.search(version_pattern, tag).group()
+
+with open("knodle/version.py", "w") as fp:
+    fp.writelines([
+        "# this is an auto-generated file on release\n",
+        f"__version__ = '{version}'\n"
+    ])
\ No newline at end of file


Test patch:
diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml
new file mode 100644
index 00000000..5f1679fe
--- /dev/null
+++ b/.github/workflows/test_deploy.yml
@@ -0,0 +1,51 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Upload to TestPyPI
+
+# Controls when the action will run. 
+on:
+  workflow_dispatch:
+    inputs:
+      release-version:
+        description: 'Test release version tag'
+        default: 'v0.1.0'
+        required: true  # Triggers the workflow on push to the master branch
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      # Sets up python3
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      # Installs and upgrades pip, installs other dependencies and installs the package from setup.py
+      - name: "Installs and upgrades pip, installs other dependencies and installs the package from setup.py"
+        run: |
+          # Upgrade pip
+          python3 -m pip install --upgrade pip
+          # Install build deps
+          python3 -m pip install setuptools wheel twine
+
+      - name: "Tag Release"
+        run: |
+          python3 update_version_from_tag.py ${{ github.event.inputs.release-version }}
+
+      # Upload to TestPyPI
+      - name: Build and Upload to TestPyPI
+        run: |
+          python3 setup.py sdist bdist_wheel
+          python3 -m twine upload dist/*
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TEST_TOKEN }}
+          TWINE_REPOSITORY: testpypi
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/data/__init__.py b/tests/data/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/data/test_statistics.py b/tests/data/test_statistics.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/evaluation/__init__.py b/tests/evaluation/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/evaluation/majority.py b/tests/evaluation/majority.py
new file mode 100644
index 00000000..057103ad
--- /dev/null
+++ b/tests/evaluation/majority.py
@@ -0,0 +1,61 @@
+import numpy as np
+
+from knodle.evaluation.majority import majority_sklearn_report
+
+
+def test_majority_vote_no_match():
+    z = np.zeros((2, 4))
+    t = np.zeros((4, 2))
+
+    z[0, 0] = 1
+    z[1, 1] = 1
+
+    t[0, 0] = 1
+    t[1, 0] = 1
+    t[2, 0] = 1
+    t[3, 0] = 1
+
+    y = np.array([1, 1])
+    report = majority_sklearn_report(z, t, y)
+    assert report["accuracy"] == 0
+
+
+def test_majority_vote_correctness():
+    z = np.zeros((2, 4))
+    t = np.zeros((4, 2))
+
+    z[0, 0] = 1
+    z[1, 1] = 1
+
+    t[0, 0] = 1
+    t[1, 1] = 1
+    t[2, 0] = 1
+    t[3, 0] = 1
+
+    y = np.array([1, 1])
+    report = majority_sklearn_report(z, t, y)
+    assert report["accuracy"] == 0.5
+
+    y = np.array([0, 1])
+    report = majority_sklearn_report(z, t, y)
+    assert report["accuracy"] == 1
+
+
+def test_majority_vote_random_guess():
+    z = np.zeros((4, 4))
+    t = np.zeros((4, 2))
+
+    z[0, 0] = 1
+    z[1, 1] = 1
+    z[2, 0] = 1
+
+    t[0, 0] = 1
+    t[1, 1] = 1
+    t[2, 0] = 1
+    t[3, 0] = 1
+
+    # no info about sample 4, thus random value is chosen
+    # y_majority = np.array([0, 1, 0, random])
+    y = np.array([0, 1, 1, 1])
+    report = majority_sklearn_report(z, t, y)
+    assert 0.5 <= report["accuracy"] <= 0.75
diff --git a/tests/test_sample.py b/tests/test_sample.py
deleted file mode 100644
index 07d71765..00000000
--- a/tests/test_sample.py
+++ /dev/null
@@ -1,12 +0,0 @@
-# Sample Test passing with nose and pytest
-import pytest
-
-
-@pytest.fixture
-def init_sum_function():
-    from knodle.sum import test_sum_function
-    return test_sum_function
-
-
-def test_pass(init_sum_function):
-    assert init_sum_function(1,1) == 2
\ No newline at end of file
diff --git a/tests/trainer/__init__.py b/tests/trainer/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/trainer/baseline/__init__.py b/tests/trainer/baseline/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/trainer/baseline/test_baseline.py b/tests/trainer/baseline/test_baseline.py
new file mode 100644
index 00000000..0556f7a6
--- /dev/null
+++ b/tests/trainer/baseline/test_baseline.py
@@ -0,0 +1,42 @@
+import numpy as np
+import torch
+from torch.utils.data import TensorDataset
+
+from knodle.model.logistic_regression_model import LogisticRegressionModel
+from knodle.trainer.baseline.majority import MajorityVoteTrainer
+
+
+def test_train():
+    num_samples = 64
+    num_features = 16
+    num_rules = 6
+    num_classes = 2
+
+    model = LogisticRegressionModel(num_features, num_classes)
+
+    x_np = np.ones((num_samples, num_features)).astype(np.float32)
+    x_tensor = torch.from_numpy(x_np)
+    model_input_x = TensorDataset(x_tensor)
+
+    rule_matches_z = np.zeros((num_samples, num_rules))
+    rule_matches_z[0, 0] = 1
+    rule_matches_z[1:, 1] = 1
+
+    mapping_rules_labels_t = np.zeros((num_rules, num_classes))
+    mapping_rules_labels_t[:, 1] = 1
+
+    trainer = MajorityVoteTrainer(
+        model=model,
+        mapping_rules_labels_t=mapping_rules_labels_t,
+        model_input_x=model_input_x,
+        rule_matches_z=rule_matches_z,
+    )
+
+    trainer.train()
+
+    y_np = np.ones((num_samples,))
+    y_labels = TensorDataset(torch.from_numpy(y_np))
+    metrics, _ = trainer.test(model_input_x, y_labels)
+
+    # Check whether the code ran up to here
+    assert 2 == 2
diff --git a/tests/trainer/snorkel/__init__.py b/tests/trainer/snorkel/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/trainer/snorkel/test_utils.py b/tests/trainer/snorkel/test_utils.py
new file mode 100644
index 00000000..b6e9b6f0
--- /dev/null
+++ b/tests/trainer/snorkel/test_utils.py
@@ -0,0 +1,24 @@
+import numpy as np
+from knodle.trainer.snorkel.utils import z_t_matrix_to_snorkel_matrix
+
+
+def test_z_t_matrix_to_snorkel_matrix():
+    z = np.array([
+        [0, 1, 0, 0],
+        [0, 0, 1, 0]
+    ])
+
+    t = np.array([
+        [1, 0],
+        [0, 1],
+        [1, 0],
+        [0, 1]
+    ])
+
+    snorkel_gold = np.array([
+        [-1, 1, -1, -1],
+        [-1, -1, 0, -1]
+    ])
+
+    snorkel_test = z_t_matrix_to_snorkel_matrix(z, t)
+    np.testing.assert_equal(snorkel_gold, snorkel_test)
diff --git a/tests/trainer/test_auto_trainer.py b/tests/trainer/test_auto_trainer.py
new file mode 100644
index 00000000..16a5e820
--- /dev/null
+++ b/tests/trainer/test_auto_trainer.py
@@ -0,0 +1,43 @@
+import numpy as np
+import torch
+from torch.utils.data import TensorDataset
+
+from knodle.model.logistic_regression_model import LogisticRegressionModel
+from knodle.trainer.auto_trainer import AutoTrainer
+
+
+def test_auto_train():
+    num_samples = 64
+    num_features = 16
+    num_rules = 6
+    num_classes = 2
+
+    model = LogisticRegressionModel(num_features, num_classes)
+
+    x_np = np.ones((num_samples, num_features)).astype(np.float32)
+    x_tensor = torch.from_numpy(x_np)
+    model_input_x = TensorDataset(x_tensor)
+
+    rule_matches_z = np.zeros((num_samples, num_rules))
+    rule_matches_z[0, 0] = 1
+    rule_matches_z[1:, 1] = 1
+
+    mapping_rules_labels_t = np.zeros((num_rules, num_classes))
+    mapping_rules_labels_t[:, 1] = 1
+
+    trainer = AutoTrainer(
+        name="majority",
+        model=model,
+        mapping_rules_labels_t=mapping_rules_labels_t,
+        model_input_x=model_input_x,
+        rule_matches_z=rule_matches_z,
+    )
+
+    trainer.train()
+
+    y_np = np.ones((num_samples,))
+    y_labels = TensorDataset(torch.from_numpy(y_np))
+    metrics = trainer.test(model_input_x, y_labels)
+
+    # Check whether the code ran up to here
+    assert 2 == 2
diff --git a/tests/trainer/utils/__init__.py b/tests/trainer/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/trainer/utils/test_denoise.py b/tests/trainer/utils/test_denoise.py
new file mode 100644
index 00000000..e4d2e131
--- /dev/null
+++ b/tests/trainer/utils/test_denoise.py
@@ -0,0 +1,52 @@
+import pytest
+import numpy as np
+from numpy.testing import assert_array_equal
+
+from knodle.trainer.utils.denoise import activate_neighbors
+
+
+def test_denoise_knn():
+    ## case 1 ##
+    test_array = np.array([[0, 1], [1, 0]])
+    right_result = np.array([[1, 1], [1, 1]])
+    indices = np.array([[0, 1], [1, 0]])
+
+    denoised_z = activate_neighbors(test_array, indices)
+    assert_array_equal(denoised_z, right_result)
+
+    # make sure all old matches persist and have same match values (currently always 1)
+    assert denoised_z[test_array == 1].sum() == test_array[test_array == 1].sum()
+
+    ## case 2 ##
+    test_array = np.diag(np.ones((3,)))
+    indices = np.array([
+        np.array([0, 1, 2]),
+        np.array([0]),
+        np.array([1])
+    ])
+
+    right_result = np.diag(np.ones((3,)))
+    right_result[0, :] = 1
+    right_result[1, 0] = 1
+    right_result[2, 1] = 1
+
+    denoised_z = activate_neighbors(test_array, indices)
+    assert_array_equal(denoised_z, right_result)
+    assert denoised_z[test_array == 1].sum() == test_array[test_array == 1].sum()
+
+    ## case 3 ##
+    test_array = np.diag(np.ones((3,)))
+    indices = np.array([
+        np.array([0, 1]),
+        np.array([1, 2]),
+        np.array([0, 2])
+    ])
+
+    right_result = np.ones((3, 3))
+    right_result[0, 2] = 0
+    right_result[1, 0] = 0
+    right_result[2, 1] = 0
+
+    denoised_z = activate_neighbors(test_array, indices)
+    assert_array_equal(denoised_z, right_result)
+    assert denoised_z[test_array == 1].sum() == test_array[test_array == 1].sum()
diff --git a/tests/trainer/utils/test_utils.py b/tests/trainer/utils/test_utils.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/trainer/wscrossweigh/__init__.py b/tests/trainer/wscrossweigh/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/trainer/wscrossweigh/test_cw_labels_calculation.py b/tests/trainer/wscrossweigh/test_cw_labels_calculation.py
new file mode 100644
index 00000000..ba37e592
--- /dev/null
+++ b/tests/trainer/wscrossweigh/test_cw_labels_calculation.py
@@ -0,0 +1,125 @@
+import pytest
+import numpy as np
+import torch
+from torch.utils.data import TensorDataset
+
+from knodle.model.logistic_regression_model import LogisticRegressionModel
+from knodle.trainer.wscrossweigh.wscrossweigh_weights_calculator import WSCrossWeighWeightsCalculator
+
+
+@pytest.fixture(scope='session')
+def get_test_data():
+    model = LogisticRegressionModel(8, 3)
+    rule_assignments_t = np.array([[1, 0, 0],
+                                   [1, 0, 0],
+                                   [0, 1, 0],
+                                   [0, 1, 0],
+                                   [0, 0, 1],
+                                   [0, 0, 1],
+                                   [0, 0, 1]])
+
+    inputs_x = TensorDataset(torch.Tensor(np.array([[1, 1, 1, 1, 1, 1, 1, 1],
+                                                    [2, 2, 2, 2, 2, 2, 2, 2],
+                                                    [3, 3, 3, 3, 3, 3, 3, 3],
+                                                    [4, 4, 4, 4, 4, 4, 4, 4],
+                                                    [5, 5, 5, 5, 5, 5, 5, 5],
+                                                    [6, 6, 6, 6, 6, 6, 6, 6],
+                                                    [7, 7, 7, 7, 7, 7, 7, 7],
+                                                    [8, 8, 8, 8, 8, 8, 8, 8]])))
+
+    rule_matches_z = np.array([[0, 1, 1, 0, 0, 0, 0],
+                               [1, 0, 1, 1, 0, 0, 0],
+                               [0, 0, 0, 0, 1, 0, 0],
+                               [0, 0, 0, 0, 0, 0, 1],
+                               [0, 0, 0, 0, 0, 1, 0],
+                               [1, 0, 1, 0, 0, 0, 0],
+                               [1, 1, 1, 1, 0, 0, 0],
+                               [0, 0, 0, 1, 0, 0, 0]])
+
+    rules_samples_ids_dict = {0: {1, 5, 6},
+                              1: {0, 6},
+                              2: {0, 1, 5, 6},
+                              3: {1, 6, 7},
+                              4: {2},  # no rel sample
+                              5: {4},  # no rel sample
+                              6: {3}}  # no rel sample
+
+    trainer = WSCrossWeighWeightsCalculator(
+        model=model,
+        mapping_rules_labels_t=rule_assignments_t,
+        model_input_x=inputs_x,
+        rule_matches_z=rule_matches_z
+    )
+
+    return trainer, rules_samples_ids_dict
+
+
+@pytest.fixture(scope='session')
+def get_cw_data_test(get_test_data):
+    # no filtering, no no_match samples
+    test_rules_idx = np.array([1, 2])
+    labels = np.array([[0.5, 0.5, 0], [0.3, 0.7, 0], [0, 0, 1], [0, 0, 1],
+                       [0, 0, 1], [0.5, 0.5, 0], [0.5, 0.5, 0], [0, 1, 0]])
+
+    return [[get_test_data[0],
+             labels,
+             test_rules_idx,
+             get_test_data[1],
+             torch.Tensor(np.array([[1, 1, 1, 1, 1, 1, 1, 1],
+                                    [2, 2, 2, 2, 2, 2, 2, 2],
+                                    [6, 6, 6, 6, 6, 6, 6, 6],
+                                    [7, 7, 7, 7, 7, 7, 7, 7]])),
+             np.array([[0.5, 0.5, 0], [0.3, 0.7, 0], [0.5, 0.5, 0], [0.5, 0.5, 0]]),
+             np.array([0, 1, 5, 6])
+             ]]
+
+
+@pytest.fixture(scope='session')
+def get_cw_data_train(get_test_data):
+    # rules to be filtered: 1, 2, no_match samples should be added
+
+    test_samples_idx = np.array([0, 6, 1, 5])
+    train_rules_idx = np.array([0, 3, 4, 5, 6])  # 1, 5, 6, 7 --> delete inters --> 7 --> + no rel --> 2, 3, 4, 7
+    labels = np.array([[0.5, 0.5, 0], [0.3, 0.7, 0], [0, 0, 1], [0, 0, 1],
+                       [0, 0, 1], [0.5, 0.5, 0], [0.5, 0.5, 0], [0, 1, 0]])
+
+    return [[get_test_data[0],
+             labels,
+             train_rules_idx,
+             get_test_data[1],
+             test_samples_idx,
+             torch.Tensor(np.array([[3, 3, 3, 3, 3, 3, 3, 3],
+                                    [4, 4, 4, 4, 4, 4, 4, 4],
+                                    [5, 5, 5, 5, 5, 5, 5, 5],
+                                    [8, 8, 8, 8, 8, 8, 8, 8]])),
+             np.array([[0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0]]),
+             np.array([2, 3, 4, 7])
+             ]]
+
+
+def test_sample_ids_matched_rules_correspondence(get_test_data):
+    # for data in get_sample_ids_matched_rules_correspondence:
+    #     assert WSCrossWeighWeightsCalculator._get_rules_samples_ids_dict(data[0]) == data[1]
+    assert WSCrossWeighWeightsCalculator._get_rules_samples_ids_dict(get_test_data[0]) == get_test_data[1]
+
+
+def test_get_cw_data_test(get_cw_data_test):
+    for data in get_cw_data_test:
+        samples, labels, ids = WSCrossWeighWeightsCalculator._get_cw_samples_labels_idx(
+            data[0], data[1], data[2], data[3]
+        )
+
+        assert torch.equal(samples.tensors[0], data[4])
+        np.testing.assert_array_equal(labels, data[5])
+        np.testing.assert_array_equal(ids, data[6])
+
+
+def test_get_cw_data_train(get_cw_data_train):
+    for data in get_cw_data_train:
+        samples, labels, ids = WSCrossWeighWeightsCalculator._get_cw_samples_labels_idx(
+            data[0], data[1], data[2], data[3], data[4]
+        )
+
+        assert torch.equal(samples.tensors[0], data[5])
+        np.testing.assert_array_equal(labels, data[6])
+        np.testing.assert_array_equal(ids, data[7])
diff --git a/tests/transformation/__init__.py b/tests/transformation/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/transformation/test_filter.py b/tests/transformation/test_filter.py
new file mode 100644
index 00000000..2acc8afe
--- /dev/null
+++ b/tests/transformation/test_filter.py
@@ -0,0 +1,36 @@
+import numpy as np
+import torch
+from torch.utils.data import TensorDataset
+
+from knodle.transformation.filter import filter_empty_probabilities
+
+
+def test_filter_empty_probabilities():
+    input_ids = np.ones((3, 4))
+    input_ids[0, 0] = 0
+    input_mask = np.ones((3, 4))
+    input_mask[1, 1] = 0
+    class_probs = np.array([
+        [0.5, 0.5],
+        [0.3, 0.7],
+        [0.0, 0.0]
+    ])
+
+    gold_ids = np.ones((2, 4))
+    gold_ids[0, 0] = 0
+    gold_mask = np.ones((2, 4))
+    gold_mask[1, 1] = 0
+    gold_probs = np.array([
+        [0.5, 0.5],
+        [0.3, 0.7]
+    ])
+
+    input_dataset = TensorDataset(torch.from_numpy(input_ids), torch.from_numpy(input_mask))
+
+    new_input_dataset, new_probs = filter_empty_probabilities(input_dataset, class_probs)
+
+    assert np.array_equal(new_input_dataset.tensors[0].detach().numpy(), gold_ids)
+    assert np.array_equal(new_input_dataset.tensors[1].detach().numpy(), gold_mask)
+    assert np.array_equal(new_probs, gold_probs)
+
+
diff --git a/tests/transformation/test_majority.py b/tests/transformation/test_majority.py
new file mode 100644
index 00000000..6bcd3595
--- /dev/null
+++ b/tests/transformation/test_majority.py
@@ -0,0 +1,98 @@
+import pytest
+
+import numpy as np
+
+from knodle.transformation.majority import (
+    probabilies_to_majority_vote, z_t_matrices_to_majority_vote_probs, z_t_matrices_to_majority_vote_labels
+)
+
+
+def test_probabilies_to_majority_vote_fixed():
+    # format: (probabilities, gold_result, settings)
+    probs_gold_result_settings = [
+        (np.array([0.5, 0.2, 0.3]), 0, {"choose_random_label": False, "other_class_id": -1}),
+        (np.array([0.5, 0.2, 0.3]), 0, {"choose_random_label": True, "other_class_id": None}),
+        (np.array([0.5, 0.2, 0.3]), 0, {"choose_random_label": False, "other_class_id": -1}),
+        (np.array([0.5, 0.2, 0.3]), 0, {"choose_random_label": False, "other_class_id": None}),
+
+        (np.array([0.5, 0.5, 0.0]), -1, {"choose_random_label": None, "other_class_id": -1}),
+        (np.array([0.0, 0.0, 0.0]), -1, {"choose_random_label": None, "other_class_id": -1})
+    ]
+
+    for probs, gold_label, settings in probs_gold_result_settings:
+        result = probabilies_to_majority_vote(probs, **settings)
+
+        assert isinstance(result, int)
+        assert result == gold_label
+
+
+def test_probabilies_to_majority_vote_random():
+    # format: (probabilities, gold_result, settings)
+    probs_gold_result_settings = [
+        (np.array([0.5, 0.5, 0.0]), [0, 1], {"choose_random_label": True, "other_class_id": None}),
+
+        (np.array([0.3, 0.3, 0, 2, 0.3]), [0, 1, 3], {"choose_random_label": True, "other_class_id": None}),
+
+        (np.array([0.0, 0.0, 0.0]), [-1], {"choose_random_label": False, "other_class_id": -1}),
+        (np.array([0.0, 0.0, 0.0]), [0, 1, 2], {"choose_random_label": True, "other_class_id": None})
+    ]
+
+    for probs, gold_label, settings in probs_gold_result_settings:
+        result = probabilies_to_majority_vote(probs, **settings)
+
+        assert isinstance(result, int)
+        assert result in gold_label
+
+
+def test_probabilies_to_majority_vote_errors():
+    probs, gold_label, settings = (
+        np.array([0.0, 0.0, 0.0]), [], {"choose_random_label": False, "other_class_id": None}
+    )
+
+    with pytest.raises(ValueError):
+        result = probabilies_to_majority_vote(probs, **settings)
+
+
+@pytest.fixture
+def prob_values():
+    z = np.zeros((4, 4))
+    t = np.zeros((4, 2))
+
+    z[0, 0] = 1
+    z[0, 2] = 1
+    z[1, 0] = 1
+    z[1, 1] = 1
+    z[1, 2] = 1
+    z[1, 3] = 1
+    z[2, 1] = 1
+
+    t[0, 0] = 1
+    t[1, 1] = 1
+    t[2, 1] = 1
+    t[3, 1] = 1
+
+    gold_probs = np.array([
+        [0.5, 0.5],
+        [0.25, 0.75],
+        [0, 1],
+        [0, 0]
+    ])
+
+    gold_labels = np.array([-1, 1, 1, -1])
+
+    return z, t, gold_probs, gold_labels
+
+
+def test_get_majority_vote_labels(prob_values):
+    z, t, _, gold_labels = prob_values
+
+    majority_labels = z_t_matrices_to_majority_vote_labels(z, t, choose_random_label=False, other_class_id=-1)
+    print(majority_labels)
+    assert np.array_equal(majority_labels, gold_labels)
+
+
+def test_get_majority_vote_probs(prob_values):
+    z, t, gold_probs, _ = prob_values
+
+    majority_probs = z_t_matrices_to_majority_vote_probs(z, t)
+    assert np.array_equal(gold_probs, majority_probs)
\ No newline at end of file
diff --git a/tests/transformation/test_rule_label_format.py b/tests/transformation/test_rule_label_format.py
new file mode 100644
index 00000000..f9daedb6
--- /dev/null
+++ b/tests/transformation/test_rule_label_format.py
@@ -0,0 +1,17 @@
+import numpy as np
+
+from knodle.transformation.rule_label_format import transform_snorkel_matrix_to_z_t
+
+
+def test_transform_snorkel_matrix_to_z_t():
+
+    lambda_matrix = np.array([[-1, 2, -1, 1], [1, -1, -1, 1], [1, 2, 0, -1]])
+
+    correct_z_matrix = np.array([[0, 1, 0, 1], [1, 0, 0, 1], [1, 1, 1, 0]])
+
+    correct_t_matrix = np.array([[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0]])
+
+    z, t = transform_snorkel_matrix_to_z_t(lambda_matrix)
+
+    np.array_equal(correct_z_matrix, z)
+    np.array_equal(correct_t_matrix, t)
diff --git a/tests/transformation/torch_input.py b/tests/transformation/torch_input.py
new file mode 100644
index 00000000..59854fd4
--- /dev/null
+++ b/tests/transformation/torch_input.py
@@ -0,0 +1,21 @@
+import numpy as np
+from numpy.testing import assert_array_equal
+
+from torch import Tensor, equal
+from torch.utils.data import TensorDataset
+
+from knodle.transformation.torch_input import input_labels_to_tensordataset
+
+
+def test_input_labels_to_tensordataset():
+
+    a = np.ones((4, 4))
+    b = np.ones((4, 3))
+    labels = np.ones((4,))
+
+    input_data = TensorDataset(Tensor(a), Tensor(b))
+
+    input_label_dataset = input_labels_to_tensordataset(input_data, labels)
+
+    assert len(input_label_dataset.tensors) == 3
+    assert_array_equal(input_label_dataset.tensors[-1].cpu().numpy(), labels)
\ No newline at end of file