Repository: HLR/DomiKnowS (language: python)

Issue:
Hotfix/sample loss



Discussion / hints:

Fix patch:
diff --git a/examples/AnimalAndFlower/main.py b/examples/AnimalAndFlower/main.py
index 313e49ba9..b36f326dc 100644
--- a/examples/AnimalAndFlower/main.py
+++ b/examples/AnimalAndFlower/main.py
@@ -14,6 +14,8 @@
 import logging, random
 import torch
 
+from regr.utils import setProductionLogMode
+
 parser = argparse.ArgumentParser()
 parser.add_argument('--cuda', dest='cuda_number', default=0, help='cuda number to train the models on',type=int)
 parser.add_argument('--solver', help='the model solver', default='primal_dual')
@@ -30,6 +32,9 @@
 # TODO add  model size and other things here
 args = parser.parse_args()
 
+# Disable Logging  
+productionMode = True  
+    
 import math
 def main():
     if torch.cuda.is_available():
@@ -38,6 +43,10 @@ def main():
         device='cpu'
     print("selected device is:",device)
 
+    
+    if productionMode:
+        setProductionLogMode()
+        
     torch.manual_seed(777)
     random.seed(777)
     logging.basicConfig(level=logging.INFO)
diff --git a/examples/WIQA/.gitignore b/examples/WIQA/.gitignore
index 6d9594aa4..c420a26b6 100644
--- a/examples/WIQA/.gitignore
+++ b/examples/WIQA/.gitignore
@@ -1,1 +1,2 @@
 /logs/
+/new_domi_1
diff --git a/examples/conll04/main-bert.py b/examples/conll04/main-bert.py
index 3dce6c2aa..ce2a44e20 100644
--- a/examples/conll04/main-bert.py
+++ b/examples/conll04/main-bert.py
@@ -6,16 +6,17 @@
 
 import numpy as np
 
-from regr.program import POIProgram, SolverPOIProgram, IMLProgram, CallbackProgram
+from regr.program import POIProgram, SolverPOIProgram, IMLProgram, CallbackProgram, SolverPOIDictLossProgram
 from regr.program.callbackprogram import ProgramStorageCallback
 from regr.program.metric import MacroAverageTracker, PRF1Tracker, DatanodeCMMetric
-from regr.program.loss import NBCrossEntropyLoss, NBCrossEntropyIMLoss
+from regr.program.loss import NBCrossEntropyLoss, NBCrossEntropyIMLoss, NBCrossEntropyDictLoss
 from regr.sensor.pytorch.sensors import FunctionalSensor, JointSensor, ModuleSensor, ReaderSensor, JointReaderSensor, FunctionalReaderSensor, cache, TorchCache
 from regr.sensor.pytorch.learners import ModuleLearner
 from regr.sensor.pytorch.relation_sensors import CompositionCandidateSensor, CompositionCandidateReaderSensor, EdgeSensor
 from regr.sensor.pytorch.query_sensor import DataNodeReaderSensor
 
 from conll.data.data import SingletonDataLoader
+from regr.utils import setProductionLogMode
 
 
 import spacy
@@ -85,7 +86,7 @@ def __init__(self, in_features) -> None:
         super().__init__(linear)
 
 
-def model():
+def model(device=None):
     from graph import graph, sentence, word, phrase, pair
     from graph import people, organization, location, other, o
     from graph import work_for, located_in, live_in, orgbase_on, kill
@@ -220,6 +221,9 @@ def find(arg1m, arg2m, data):
     class Program(CallbackProgram, SolverPOIProgram):
         pass
 
+    class DictCallBackProgram(CallbackProgram, SolverPOIDictLossProgram):
+        pass
+
     lbp1 = Program(
         graph, poi=(phrase, pair), inferTypes=['ILP', 'local/argmax'],
         loss=MacroAverageTracker(NBCrossEntropyLoss()),
@@ -233,10 +237,28 @@ class Program(CallbackProgram, SolverPOIProgram):
         metric={
             'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))})
 
+    lbp = DictCallBackProgram(
+        graph, poi=(phrase, pair), inferTypes=['local/argmax'],
+        dictloss={
+            str(o.name): NBCrossEntropyDictLoss(weight=torch.tensor([ 4.3836,  0.5644]).to(device)),
+            str(location.name): NBCrossEntropyDictLoss(weight=torch.tensor([ 0.5197, 13.1628]).to(device)),
+            str(people.name): NBCrossEntropyDictLoss(weight=torch.tensor([ 0.5168, 15.3461]).to(device)), 
+            str(other.name): NBCrossEntropyDictLoss(weight=torch.tensor([ 0.5122, 21.0087]).to(device)),             
+            str(organization.name): NBCrossEntropyDictLoss(weight=torch.tensor([ 0.5100, 25.3879]).to(device)), 
+            str(work_for.name): NBCrossEntropyDictLoss(weight=torch.tensor([0.6653, 2.0123]).to(device)), 
+            str(located_in.name): NBCrossEntropyDictLoss(weight=torch.tensor([0.6468, 2.2027]).to(device)), 
+            str(live_in.name): NBCrossEntropyDictLoss(weight=torch.tensor([0.6269, 2.4697]).to(device)), 
+            str(orgbase_on.name): NBCrossEntropyDictLoss(weight=torch.tensor([0.6128, 2.7167]).to(device)), 
+            str(kill.name): NBCrossEntropyDictLoss(weight=torch.tensor([0.5801, 3.6222]).to(device)), 
+            "default": NBCrossEntropyDictLoss()},
+        metric={
+            # 'ILP': PRF1Tracker(DatanodeCMMetric()),
+            'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))})
+
     return lbp, lbp1
 
 def main(args):
-    program, program1 = model()
+    program, program1 = model(device=args.gpu)
 
     split_id = args.split
     print(args.number)
@@ -258,8 +280,8 @@ def save_epoch(program, epoch=1):
     def compute_scores(item, criteria="P"):
         entities = ["location", "people", "organization", "other"]
         relations = ["work_for", "located_in", "live_in", "orgbase_on", "kill"]
-        instances = {"location": 931, "people": 793, "organization": 523, "other": 572, "work_for": 94, "located_in": 107, "live_in": 108, 
-                     "orgbase_on": 93, "kill": 53}
+        instances = {"location": 904, "people": 781, "organization": 482, "other": 656, "work_for": 55, "located_in": 88, "live_in": 129, 
+                     "orgbase_on": 102, "kill": 39}
         sum_entity = 0
         sum_relations = 0
         precision_entity = 0
@@ -270,14 +292,14 @@ def compute_scores(item, criteria="P"):
         precision_all = 0
         normal_precision_all = 0
         for key in entities:
-            sum_entity += instances[key]
-            precision_entity += instances[key] * item[key][criteria]
-            normal_precision_entity += item[key][criteria]
+            sum_entity += float(instances[key])
+            precision_entity += float(instances[key]) * float(item[key][criteria])
+            normal_precision_entity += float(item[key][criteria])
 
         for key in relations:
-            sum_relations += instances[key]
-            precision_relations += instances[key] * item[key][criteria]
-            normal_precision_relations += item[key][criteria]
+            sum_relations += float(instances[key])
+            precision_relations += float(instances[key]) * float(item[key][criteria])
+            normal_precision_relations += float(item[key][criteria])
 
         sum_all = sum_relations + sum_entity
         precision_all = precision_entity + precision_relations
@@ -355,7 +377,7 @@ def parse_arguments():
         type=float,
         required=False,
         choices=[1, 0.25, 0.1],
-        default=1,
+        default=0.25,
     )
     parser.add_argument(
         "-i",
@@ -363,7 +385,7 @@ def parse_arguments():
         help="Number of iterations",
         type=int,
         required=False,
-        default=10,
+        default=100,
     )
     parser.add_argument(
         "-l",
@@ -389,7 +411,7 @@ def parse_arguments():
         help="GPU option",
         type=str,
         required=False,
-        default="auto",
+        default="cuda:1",
         choices=[
             "auto",
             "cpu",
@@ -411,4 +433,5 @@ def parse_arguments():
 
 if __name__ == '__main__':
     args = parse_arguments()
+    setProductionLogMode()
     main(args)
\ No newline at end of file
diff --git a/examples/simple-logic/main_pd_sample.py b/examples/simple-logic/main_pd_sample.py
index f2f35a1a0..82b97b805 100644
--- a/examples/simple-logic/main_pd_sample.py
+++ b/examples/simple-logic/main_pd_sample.py
@@ -67,7 +67,7 @@ def main():
         'y0': [[1.,0.]],
         'y1': [[0.,1.]]
         }]
-    program.train(data, train_epoch_num=20, c_warmup_iters=0, Optim=lambda param: torch.optim.SGD(param, lr=1), device='auto')
+    program.train(data, train_epoch_num=2000, c_warmup_iters=0, Optim=lambda param: torch.optim.SGD(param, lr=1), device='auto')
     print('Train loss:', program.model.loss)
 
     program.test(data)
diff --git a/examples/sudoku/main.ipynb b/examples/sudoku/main.ipynb
index b956a366e..a1e47f69f 100644
--- a/examples/sudoku/main.ipynb
+++ b/examples/sudoku/main.ipynb
@@ -2,16 +2,16 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 12,
-   "id": "renewable-tulsa",
+   "execution_count": 51,
+   "id": "ethical-encoding",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/home/hfaghihi/Framework/new/DomiKnowS/examples\n",
-      "root Folder Absoloute path:  /home/hfaghihi/Framework/new/DomiKnowS\n"
+      "/home/hfaghihi/Framework/fix/DomiKnowS/examples\n",
+      "root Folder Absoloute path:  /home/hfaghihi/Framework/fix/DomiKnowS\n"
      ]
     }
    ],
@@ -42,29 +42,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
-   "id": "dependent-steam",
+   "execution_count": 52,
+   "id": "checked-fight",
    "metadata": {},
    "outputs": [],
    "source": [
     "from regr.data.reader import RegrReader\n",
     "import torch\n",
-    "\n",
+    "from random import sample\n",
+    "import copy\n",
     "\n",
     "class SudokuReader(RegrReader):\n",
-    "    def parse_file(self):\n",
-    "        return [[\"random\"] * 1]\n",
-    "    def getidval(self, item):\n",
-    "        return [1]\n",
-    "    def getwhole_sudokuval(self, item):\n",
+    "    def __init__(self, *args, **kargs):\n",
+    "        super().__init__(*args, *kargs)\n",
     "        base  = 3\n",
-    "        side  = base*base\n",
+    "        self.side  = base*base\n",
     "\n",
     "        # pattern for a baseline valid solution\n",
-    "        def pattern(r,c): return (base*(r%base)+r//base+c)%side\n",
+    "        def pattern(r,c): return (base*(r%base)+r//base+c)%self.side\n",
     "\n",
     "        # randomize rows, columns and numbers (of valid base pattern)\n",
-    "        from random import sample\n",
     "        def shuffle(s): return sample(s,len(s)) \n",
     "        rBase = range(base) \n",
     "        rows  = [ g*base + r for g in shuffle(rBase) for r in shuffle(rBase) ] \n",
@@ -73,11 +70,32 @@
     "\n",
     "        # produce board using randomized baseline pattern\n",
     "        board = [ [nums[pattern(r,c)] for c in cols] for r in rows ]\n",
-    "        squares = side*side\n",
+    "        self._sudoku = board\n",
+    "        \n",
+    "        board1 = copy.copy(self._sudoku)\n",
+    "        squares = self.side*self.side\n",
     "        empties = squares * 3//4\n",
+    "        F = []\n",
+    "        for i in board1:\n",
+    "            F.append([])\n",
+    "            for j in i:\n",
+    "                F[-1].append(j)\n",
+    "                \n",
     "        for p in sample(range(squares),empties):\n",
-    "            board[p//side][p%side] = 0\n",
-    "        return torch.tensor(board)\n",
+    "            F[p//self.side][p%self.side] = 0\n",
+    "        self.F = F\n",
+    "        \n",
+    "    def parse_file(self):\n",
+    "        return [[\"random\"] * 1]\n",
+    "    \n",
+    "    def getidval(self, item):\n",
+    "        return [1]\n",
+    "    \n",
+    "    def getwhole_sudokuval(self, item):\n",
+    "        return torch.tensor(self.F)\n",
+    "    \n",
+    "    def getsudokuval(self, item):\n",
+    "        return self._sudoku\n",
     "            \n",
     "    \n",
     "    def getsizeval(self, item):\n",
@@ -87,51 +105,60 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "id": "unsigned-helicopter",
+   "execution_count": 53,
+   "id": "diverse-barbados",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainreader = SudokuReader(\"randn\", type=\"raw\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "id": "increasing-berlin",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'id': [1], 'size': (9, 9), 'whole_sudoku': tensor([[8, 6, 5, 0, 0, 0, 3, 7, 0],\n",
-      "        [0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
-      "        [0, 0, 0, 6, 0, 0, 0, 0, 0],\n",
-      "        [4, 0, 0, 0, 0, 6, 0, 0, 0],\n",
-      "        [0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
-      "        [0, 0, 0, 3, 4, 1, 0, 0, 0],\n",
-      "        [1, 0, 0, 0, 6, 0, 9, 0, 4],\n",
-      "        [9, 0, 0, 7, 1, 8, 6, 0, 0],\n",
-      "        [0, 0, 0, 0, 0, 0, 0, 0, 0]])}\n",
-      "tensor([[0, 0],\n",
-      "        [0, 1],\n",
-      "        [0, 2],\n",
-      "        [0, 6],\n",
-      "        [0, 7],\n",
-      "        [1, 3],\n",
-      "        [2, 3],\n",
-      "        [3, 0],\n",
-      "        [3, 5],\n",
-      "        [5, 3],\n",
-      "        [5, 4],\n",
+      "{'id': [1], 'size': (9, 9), 'sudoku': [[3, 7, 4, 2, 5, 9, 6, 1, 8], [6, 8, 1, 7, 4, 3, 9, 5, 2], [9, 2, 5, 8, 1, 6, 3, 4, 7], [1, 6, 2, 3, 8, 4, 5, 7, 9], [4, 3, 8, 9, 7, 5, 1, 2, 6], [5, 9, 7, 6, 2, 1, 4, 8, 3], [2, 1, 9, 4, 6, 8, 7, 3, 5], [7, 5, 3, 1, 9, 2, 8, 6, 4], [8, 4, 6, 5, 3, 7, 2, 9, 1]], 'whole_sudoku': tensor([[0, 0, 4, 0, 5, 0, 0, 0, 0],\n",
+      "        [6, 0, 0, 0, 0, 3, 9, 0, 0],\n",
+      "        [0, 0, 0, 0, 1, 0, 3, 0, 7],\n",
+      "        [0, 0, 2, 0, 0, 0, 0, 7, 0],\n",
+      "        [0, 3, 8, 9, 0, 0, 0, 0, 0],\n",
+      "        [5, 0, 0, 0, 0, 1, 0, 0, 0],\n",
+      "        [2, 0, 0, 0, 0, 0, 0, 3, 0],\n",
+      "        [7, 0, 0, 1, 0, 0, 0, 0, 0],\n",
+      "        [8, 0, 0, 0, 0, 0, 2, 0, 0]])}\n",
+      "tensor([[0, 2],\n",
+      "        [0, 4],\n",
+      "        [1, 0],\n",
+      "        [1, 5],\n",
+      "        [1, 6],\n",
+      "        [2, 4],\n",
+      "        [2, 6],\n",
+      "        [2, 8],\n",
+      "        [3, 2],\n",
+      "        [3, 7],\n",
+      "        [4, 1],\n",
+      "        [4, 2],\n",
+      "        [4, 3],\n",
+      "        [5, 0],\n",
       "        [5, 5],\n",
       "        [6, 0],\n",
-      "        [6, 4],\n",
-      "        [6, 6],\n",
-      "        [6, 8],\n",
+      "        [6, 7],\n",
       "        [7, 0],\n",
       "        [7, 3],\n",
-      "        [7, 4],\n",
-      "        [7, 5],\n",
-      "        [7, 6]])\n",
-      "(tensor([0, 0, 0, 0, 0, 1, 2, 3, 3, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7]), tensor([0, 1, 2, 6, 7, 3, 3, 0, 5, 3, 4, 5, 0, 4, 6, 8, 0, 3, 4, 5, 6]))\n",
-      "tensor([8, 6, 5, 3, 7, 1, 6, 4, 6, 3, 4, 1, 1, 6, 9, 4, 9, 7, 1, 8, 6])\n"
+      "        [8, 0],\n",
+      "        [8, 6]])\n",
+      "(tensor([0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8]), tensor([2, 4, 0, 5, 6, 4, 6, 8, 2, 7, 1, 2, 3, 0, 5, 0, 7, 0, 3, 0, 6]))\n",
+      "tensor([4, 5, 6, 3, 9, 1, 3, 7, 2, 7, 3, 8, 9, 5, 1, 2, 3, 7, 1, 8, 2])\n"
      ]
     }
    ],
    "source": [
-    "trainreader = SudokuReader(\"randn\", type=\"raw\")\n",
     "item = next(iter(trainreader))\n",
     "print(item)\n",
     "indices = (item['whole_sudoku'] != 0).nonzero()\n",
@@ -143,14 +170,14 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "revised-ecuador",
+   "id": "adaptive-spiritual",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Log file for dataNode is in: /home/hfaghihi/Framework/new/DomiKnowS/examples/sudoku/logs/datanode.log\n",
+      "Log file for dataNode is in: /home/hfaghihi/Framework/fix/DomiKnowS/examples/sudoku/logs/datanode.log\n",
       "col1\n"
      ]
     },
@@ -161,7 +188,7 @@
       "main.py:26: UserWarning: Please use OrderedDict rather than dict to prevent unpredictable order of arguments.For this instance, OrderedDict([('arg1', 'empty_entry'), ('arg2', 'empty_entry')]) is used.\n",
       "  \n",
       "main.py:31: UserWarning: Please use OrderedDict rather than dict to prevent unpredictable order of arguments.For this instance, OrderedDict([('col1', 'empty_entry'), ('col2', 'empty_entry')]) is used.\n",
-      "  return [1]\n",
+      "  class SudokuReader(RegrReader):\n",
       "main.py:36: UserWarning: Please use OrderedDict rather than dict to prevent unpredictable order of arguments.For this instance, OrderedDict([('entry1', 'empty_entry'), ('entry2', 'empty_entry')]) is used.\n",
       "  # pattern for a baseline valid solution\n"
      ]
@@ -254,7 +281,7 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "subtle-wallace",
+   "id": "advisory-edmonton",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -310,7 +337,7 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "authentic-leeds",
+   "id": "formal-double",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -409,7 +436,7 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "id": "underlying-korean",
+   "id": "material-black",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -442,14 +469,15 @@
     "        \n",
     "        sample = True,\n",
     "        sampleSize=300, \n",
-    "        sampleGlobalLoss = True\n",
+    "        sampleGlobalLoss = False,\n",
+    "        beta=10\n",
     "        )"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "id": "spatial-replacement",
+   "execution_count": 8,
+   "id": "improving-richards",
    "metadata": {},
    "outputs": [
     {
@@ -459,1419 +487,47 @@
       "INFO:regr.program.program:Epoch: 1\n",
       "INFO:regr.program.program:Training:\n",
       "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 2\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 3\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 4\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 5\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 6\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 7\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 8\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 9\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 10\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 11\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 12\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 13\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 14\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 15\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 16\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 17\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 18\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 19\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 20\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 21\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 22\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 23\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 24\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 25\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 26\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 27\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 28\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 29\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 30\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 31\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 32\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 33\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 34\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 35\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 36\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
+      "INFO:regr.program.program:{'empty_entry_label': tensor(2.2740)}\n",
+      "INFO:regr.program.program: - Constraint loss:\n",
+      "INFO:regr.program.program:{'LC5': tensor(-5.7038), 'LC9': tensor(-5.7038), 'LC13': tensor(-5.7038), 'LC17': tensor(-5.7038), 'LC21': tensor(-5.7038), 'LC25': tensor(-5.7038), 'LC29': tensor(-5.7038), 'LC33': tensor(-5.7038), 'LC37': tensor(-5.7038), 'LC41': tensor(-5.7038), 'LC45': tensor(-5.7038), 'LC49': tensor(-5.7038), 'LC53': tensor(-5.7038), 'LC57': tensor(-5.7038), 'LC61': tensor(-5.7038), 'LC65': tensor(-5.7038), 'LC69': tensor(-5.7038), 'LC73': tensor(-5.7038), 'LC77': tensor(-5.7038), 'LC81': tensor(-5.7038), 'LC85': tensor(-5.7038), 'LC89': tensor(-5.7038), 'LC93': tensor(-5.7038), 'LC97': tensor(-5.7038), 'LC101': tensor(-5.7038), 'LC105': tensor(-5.7038), 'LC109': tensor(-5.7038)}\n",
       "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 37\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 38\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 39\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 40\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 41\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 42\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 43\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 44\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 45\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 46\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 47\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 48\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 49\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 50\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 51\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 52\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 53\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 54\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 55\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 56\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 57\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 58\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 59\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 60\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 61\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 62\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 63\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 64\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 65\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 66\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 67\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 68\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 69\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 70\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 71\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 72\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 73\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 74\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 75\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 76\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 77\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 78\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 79\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 80\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 81\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 82\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 83\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 84\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 85\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 86\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 87\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 88\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 89\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 90\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 91\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 92\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 93\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 94\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 95\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 96\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 97\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 98\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 99\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 100\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 101\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 102\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 103\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 104\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 105\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 106\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 107\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 108\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 109\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 110\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 111\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 112\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 113\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 114\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 115\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 116\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 117\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 118\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 119\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 120\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 121\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 122\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 123\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 124\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 125\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 126\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 127\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 128\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 129\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 130\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 131\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 132\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 133\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 134\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 135\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 136\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 137\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 138\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 139\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 140\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 141\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 142\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 143\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 144\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 145\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 146\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 147\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 148\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 149\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 150\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 151\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 152\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 153\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 154\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 155\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 156\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 157\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 158\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 159\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 160\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 161\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 162\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 163\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 164\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 165\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 166\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 167\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 168\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 169\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 170\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 171\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 172\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 173\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 174\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 175\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 176\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 177\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 178\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 179\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 180\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 181\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 182\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 183\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 184\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 185\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 186\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 187\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 188\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 189\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 190\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 191\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 192\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 193\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 194\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 195\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 196\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 197\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 198\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 199\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program:Epoch: 200\n",
-      "INFO:regr.program.program:Training:\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:None\n",
-      "INFO:regr.program.program: - metric:\n",
-      "INFO:regr.program.program: - - argmax\n",
-      "INFO:regr.program.program:None\n"
+      "INFO:regr.program.program: - - argmax\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "81\n"
      ]
     }
    ],
    "source": [
     "from IPython.utils import io\n",
     "\n",
-    "with io.capture_output() as captured:\n",
-    "    program.train(trainreader, train_epoch_num=200, c_warmup_iters=0, \n",
-    "                  Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')\n",
+    "for i in range(1):\n",
+    "    with io.capture_output() as captured:\n",
+    "        program.train(trainreader, train_epoch_num=1, c_warmup_iters=0, \n",
+    "                      Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')\n",
+    "    for datanode in program.populate(trainreader):\n",
+    "        count = 0\n",
+    "        _sud = list(trainreader)[0]['sudoku']\n",
+    "        entries = datanode.getChildDataNodes(conceptName=empty_entry)\n",
+    "        for entry in entries:\n",
+    "            row = entry.getAttribute('rows').item()\n",
+    "            col = entry.getAttribute('cols').item()\n",
+    "            val = entry.getAttribute(empty_entry_label, 'local/argmax').argmax(dim=-1)\n",
+    "            if val != _sud[row][col]:\n",
+    "                count += 1\n",
+    "        print(count)\n",
+    "        \n",
     "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
-   "id": "solid-patch",
+   "execution_count": 9,
+   "id": "thirty-swaziland",
    "metadata": {},
    "outputs": [
     {
@@ -1881,327 +537,327 @@
       "sodoku 0\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 0., 1., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([1., 0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([1., 0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 1., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 0., 1., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([1., 0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([1., 0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 1., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 0., 1., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 0., 1., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 1., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 0., 1., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 1., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 0., 1., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 0., 1., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 0., 0., 0., 1., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 1., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 0., 0., 0., 1., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')\n",
+      "tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.], device='cuda:0')\n",
       "---\n"
      ]
     }
@@ -2222,7 +878,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "rotary-beatles",
+   "id": "numerical-organic",
    "metadata": {},
    "outputs": [],
    "source": []
diff --git a/examples/sudoku/main_6by6.py b/examples/sudoku/main_6by6.py
new file mode 100644
index 000000000..cf5e9bcb4
--- /dev/null
+++ b/examples/sudoku/main_6by6.py
@@ -0,0 +1,502 @@
+import os,sys
+import torch
+currentdir = os.path.dirname(os.getcwd())
+print(currentdir)
+# parent_dir = os.path.abspath(os.path.join(currentdir, os.pardir))
+root = os.path.dirname(currentdir)
+print("root Folder Absolute path: ", root)
+
+sys.path.append(root)
+
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+from regr.data.reader import RegrReader
+from regr.program.lossprogram import SampleLossProgram, PrimalDualProgram
+from regr.program.model.pytorch import SolverModel
+from regr.utils import setProductionLogMode
+
+
+from random import sample
+
+class SudokuReader(RegrReader):
+    def parse_file(self):
+        # base  = 3
+        # side  = base*base
+
+        # pattern for a baseline valid solution
+        # def pattern(r,c): return (base*(r%base)+r//base+c)%side
+
+        # # randomize rows, columns and numbers (of valid base pattern)
+        # from random import sample
+        # def shuffle(s): return sample(s,len(s)) 
+        # rBase = range(base) 
+        # rows  = [ g*base + r for g in shuffle(rBase) for r in shuffle(rBase) ] 
+        # cols  = [ g*base + c for g in shuffle(rBase) for c in shuffle(rBase) ]
+        # nums  = shuffle(range(1,base*base+1))
+
+        # # produce board using randomized baseline pattern
+        # board = [ [nums[pattern(r,c)] for c in cols] for r in rows ]
+        # F = []
+        # for i in board:
+        #     F.append([])
+        #     for j in i:
+        #         F[-1].append(j)
+        # squares = side*side
+        # empties = squares * 3//4
+        # for p in sample(range(squares),empties):
+        #     board[p//side][p%side] = 0
+        # board = torch.tensor(board)
+
+        board = torch.tensor([[0, 4, 1, 0, 3, 0],
+                 [0, 0, 3, 5, 1, 0],
+                 [4, 1, 0, 0, 2, 5],
+                 [0, 3, 5, 4, 6, 0],
+                 [1, 0, 2, 6, 0, 3],
+                 [3, 0, 0, 0, 0, 2]
+                 ])
+
+        F = torch.tensor([[5, 4, 1, 2, 3, 6],
+                 [6, 2, 3, 5, 1, 4],
+                 [4, 1, 6, 3, 2, 5],
+                 [2, 3, 5, 4, 6, 1],
+                 [1, 5, 2, 6, 4, 3],
+                 [3, 6, 4, 1, 5, 2]
+                 ])
+    
+        return [{"board": board, "F": F}]
+    
+    def getidval(self, item):
+        return [1]
+    def getwhole_sudokuval(self, item):
+        return item['board']
+    
+    def getsudokuval(self, item):
+        return item['F']
+    
+    def getsizeval(self, item):
+        return 6, 6
+    
+    
+trainreader = SudokuReader("randn", type="raw")
+
+from regr.graph import Graph, Concept, Relation
+from regr.graph.logicalConstrain import andL, existsL, notL, atMostL, ifL, fixedL, eqL, exactL
+from regr.graph import EnumConcept
+
+Graph.clear()
+Concept.clear()
+Relation.clear()
+
+with Graph('global') as graph:
+    sudoku = Concept("sodoku")
+    
+#     empty_entries = Concept(name="empty_entries")
+#     fixed_entries = Concept(name="fixed_entries")
+    
+#     (sudoku_empty, sudoku_fixed) = sudoku.has_a(empty_entries, fixed_entries)
+    
+    empty_entry = Concept(name='empty_entry')
+    (empty_rel, ) = sudoku.contains(empty_entry)
+    
+#     fixed_entry = Concept(name='fixed_entry')
+#     (fixed_rel, ) = fixed_entries.contains(fixed_entry)
+    
+    same_row = Concept(name="same_row")
+#     same_row_mixed = Concept(name="same_row_mixed")
+    (same_row_arg1, same_row_arg2) = same_row.has_a(row1=empty_entry, row2=empty_entry)
+#     (same_row_mixed_arg1, same_row_mixed_arg2) = same_row_mixed.has_a(arg1=empty_entry, arg2=fixed_entry)
+    
+    same_col = Concept(name="same_col")
+#     same_col_mixed = Concept(name="same_col_mixed")
+    (same_col_arg1, same_col_arg2) = same_col.has_a(col1=empty_entry, col2=empty_entry)
+    print(same_col_arg1)
+#     (same_col_mixed_arg1, same_col_mixed_arg2) = same_col_mixed.has_a(arg1=empty_entry, arg2=fixed_entry)
+
+    same_table = Concept(name="same_table")
+    (same_table_arg1, same_table_arg2) = same_table.has_a(table1=empty_entry, table2=empty_entry)
+    
+    empty_entry_label = empty_entry(name="empty_entry_label", ConceptClass=EnumConcept, 
+                                    values=["v1", "v2", "v3", "v4", "v5", "v6"])
+    v = [getattr(empty_entry_label, a) for a in ('', *empty_entry_label.enum)]
+
+    ### Constraints
+    # entry = concept(name="entry")
+    # entry["given"] = ReaderSensor(keyword="given")
+    # entry_label= entry(name="label")
+    # fixedL(entry_label("x", eqL(entry, "given", {True})))
+    
+    FIXED = True
+    
+    # ifL(empty_entry,  atMostL(*empty_entry_label.attributes), active = True, sampleEntries = True)
+    
+    fixedL(empty_entry_label("x", eqL(empty_entry, "fixed", {True})), active = FIXED)
+    
+    for row_num in range(6):
+        # andL(*[exactL(v[i](path = (eqL(empty_entry, "rows", {row_num})))) for i in range(1, 10)])
+        # andL(*[exactL(v[i](path = (eqL(empty_entry, "cols", {row_num})))) for i in range(1, 10)])
+        # andL(*[exactL(v[i](path = (eqL(empty_entry, "tables", {row_num})))) for i in range(1, 10)])
+
+        for j in range(1, 7):
+            exactL(v[j](path = (eqL(empty_entry, "rows", {row_num}))))
+            exactL(v[j](path = (eqL(empty_entry, "cols", {row_num}))))
+            exactL(v[j](path = (eqL(empty_entry, "tables", {row_num}))))
+   
+import torch.nn as nn
+
+class Conv2dSame(torch.nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size, bias=True, padding_layer=torch.nn.ReflectionPad2d):
+        """It only support square kernels and stride=1, dilation=1, groups=1."""
+        super(Conv2dSame, self).__init__()
+        ka = kernel_size // 2
+        kb = ka - 1 if kernel_size % 2 == 0 else ka
+        self.net = nn.Sequential(
+            padding_layer((ka,kb,ka,kb)),
+            nn.Conv2d(in_channels, out_channels, kernel_size, bias=bias),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(True)
+        )
+    def forward(self, x):
+        return self.net(x)
+    
+class SudokuSolver(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.W = torch.nn.Parameter(torch.rand((36,6)))
+        
+    def __call__(self, X):
+        
+        return self.W
+    
+from regr.sensor.pytorch.sensors import JointSensor, FunctionalReaderSensor
+from regr.sensor.pytorch.learners import ModuleLearner
+from regr.sensor.pytorch.relation_sensors import CompositionCandidateSensor
+
+class JointFunctionalReaderSensor(JointSensor, FunctionalReaderSensor):
+    pass
+
+# def getempties(*prev, data):
+#     rows, cols = torch.where(data == 0)
+#     rel = torch.ones(1, 1)
+#     return [rows], [cols]
+    
+def getfixed(*prev, data):
+    rows, cols = torch.where(data != 0)
+    fix = torch.zeros(data.shape)
+    vals = torch.ones(data.shape) * -1
+    for i, j in zip(rows.detach().tolist(), cols.detach().tolist()):
+        fix[i][j] = 1
+        vals[i][j] = data[i][j]
+                
+    return fix.reshape(fix.shape[0]*fix.shape[1]), vals.reshape(vals.shape[0]*vals.shape[1])
+
+def makeSoduko(*prev, data):
+    num_rows = data[0]
+    num_cols = data[1]
+    rows = torch.arange(num_rows).unsqueeze(-1)
+    rows = rows.repeat(1,num_cols).reshape(num_rows*num_cols)
+    
+    cols = torch.arange(num_rows)
+    cols = cols.unsqueeze(0).repeat(num_rows, 1).reshape(num_rows*num_cols)
+
+    tables = []
+    passing = int(num_cols/3)
+    for i in range(data[0]*data[1]):
+        row = rows[i]
+        col = cols[i]
+        x = int((col.item() / 3))
+        x += int(row.item() / 2) * passing
+        tables.append(x)
+    tables = torch.tensor(tables)
+
+    
+    rel = torch.ones(data[0]*data[1], 1)
+    
+    return rows, cols, tables, rel
+
+def getlabel(*prev, data):
+    rows, cols = torch.where(data != 0)
+    vals = torch.ones(data.shape) * -100
+    for i, j in zip(rows.detach().tolist(), cols.detach().tolist()):
+        vals[i][j] = data[i][j] - 1
+        
+    return vals.reshape(vals.shape[0]*vals.shape[1])
+    
+def createSudoku(*prev, data):
+    return [1]
+
+sudoku['index'] = FunctionalReaderSensor(keyword='size', forward=createSudoku)
+    
+empty_entry['rows', 'cols', 'tables', empty_rel] = JointFunctionalReaderSensor(sudoku['index'], keyword='size', forward=makeSoduko)
+empty_entry['fixed', 'val'] = JointFunctionalReaderSensor('rows', 'cols', empty_rel, keyword='whole_sudoku', forward=getfixed)
+
+empty_entry[empty_entry_label] = ModuleLearner('val', module=SudokuSolver())
+empty_entry[empty_entry_label] = FunctionalReaderSensor(keyword='whole_sudoku', label=True, forward=getlabel)
+
+def filter_col(*inputs, col1, col2):
+    if col1.getAttribute('cols').item() == col2.getAttribute('cols').item() and col1.instanceID != col2.instanceID:
+        return True
+    return False
+    
+same_col[same_col_arg1.reversed, same_col_arg2.reversed] = CompositionCandidateSensor(
+    empty_entry['cols'],
+    relations=(same_col_arg1.reversed, same_col_arg2.reversed),
+    forward=filter_col)
+
+def filter_row(*inputs, row1, row2):
+    if row1.getAttribute('rows').item() == row2.getAttribute('rows').item() and row1.instanceID != row2.instanceID:
+        return True
+    return False
+    
+same_row[same_row_arg1.reversed, same_row_arg2.reversed] = CompositionCandidateSensor(
+    empty_entry['rows'],
+    relations=(same_row_arg1.reversed, same_row_arg2.reversed),
+    forward=filter_row)
+
+def filter_table(*inputs, table1, table2):
+    if table1.instanceID != table2.instanceID:
+        if int(table1.getAttribute('rows').item() / 3) == int(table2.getAttribute('rows').item() / 3) and \
+           int(table1.getAttribute('cols').item() / 3) == int(table2.getAttribute('cols').item() / 3):
+            return True
+        
+    return False
+    
+same_table[same_table_arg1.reversed, same_table_arg2.reversed] = CompositionCandidateSensor(
+    empty_entry['rows'], empty_entry['cols'],
+    relations=(same_table_arg1.reversed, same_table_arg2.reversed),
+    forward=filter_table)
+
+# fixed_entries['rows1', 'cols'] = JointFunctionalReaderSensor(keyword='whole_sudoku', forward=getfixed)
+
+### What kind of model should we use for learning the entries? Because it should be aware of all other decision to make the correct decision,
+##  otherwise it is impossible for the model to learn good weights.
+
+from regr.program import SolverPOIProgram
+from regr.program.metric import MacroAverageTracker
+from regr.program.loss import NBCrossEntropyLoss
+
+program1 = SolverPOIProgram(
+        graph, poi=(sudoku, empty_entry, same_row, same_col, same_table), inferTypes=['local/argmax', 'ILP'],
+        loss=MacroAverageTracker(NBCrossEntropyLoss()),
+#         metric={
+#             'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))}
+)
+
+program = SampleLossProgram(
+        graph, SolverModel,
+        poi=(sudoku, empty_entry, same_row, same_col, same_table),
+        inferTypes=['local/argmax'],
+        # inferTypes=['ILP', 'local/argmax'],
+#         metric={
+#             'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))},
+
+        #metric={ 'softmax' : ValueTracker(prediction_softmax),
+        #       'ILP': PRF1Tracker(DatanodeCMMetric()),
+        #        'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))
+        #       },
+        loss=MacroAverageTracker(NBCrossEntropyLoss()),
+        
+        sample = True,
+        sampleSize=2000, 
+        sampleGlobalLoss = True,
+        beta=50
+        )
+
+from regr.program.metric import PRF1Tracker, DatanodeCMMetric
+
+programRLoss = PrimalDualProgram(
+       graph, SolverModel, 
+       poi=(sudoku, empty_entry, same_row, same_col, same_table),
+       inferTypes=['local/argmax'],
+       loss=MacroAverageTracker(NBCrossEntropyLoss()),
+       metric={'softmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))}
+       )
+
+# program1 = SolverPOIProgram(
+#         graph, poi=(sudoku, empty_entry), inferTypes=['local/argmax', 'ILP'],
+#         loss=MacroAverageTracker(NBCrossEntropyLoss()),
+        # metric={
+        #     'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))}
+            # )
+
+# program = SolverPOIProgram(
+#         graph, poi=(sudoku, empty_entry, same_col, same_row, same_table), inferTypes=['ILP', 'local/argmax'],
+#         loss=MacroAverageTracker(NBCrossEntropyLoss()),
+#         metric={
+#             'ILP': PRF1Tracker(DatanodeCMMetric()),
+#             'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))})
+
+# for datanode in program.populate(trainreader):
+#     print(datanode)
+    
+#     print(datanode.getChildDataNodes(conceptName=empty_entry))
+    
+#     sudokuLoss = datanode.calculateLcLoss(sample = True, sampleSize = 100)
+    
+#     print("sudokuLoss - %s"%(sudokuLoss))
+
+
+# Disable Logging  
+productionMode = True  
+if productionMode:
+    setProductionLogMode()
+
+# program1.train(trainreader, train_epoch_num=1, Optim=lambda param: torch.optim.SGD(param, lr=1), device='auto')
+
+### test to see whether the FixedL is working, 
+def testSudokuPrediction(entries, predictionP = None):
+    
+    if predictionP != None:
+        prediction = predictionP
+    else:
+        prediction = torch.zeros((6,6))
+        for entry in entries:
+            row = entry.getAttribute('rows').item()
+            col = entry.getAttribute('cols').item()
+    
+            val = entry.getAttribute(empty_entry_label, 'ILP').argmax(dim=-1).item() + 1
+            prediction[row][col] = val
+          
+    print("Prediction:\n %s"%(prediction))
+     
+    fixedSud = torch.zeros((6,6)) #[[None for i in range(6)] for i in range(6)]
+    for entry in entries:
+        # t = entry.getAttribute(empty_entry_label, 'ILP')
+        row = entry.getAttribute('rows').item()
+        col = entry.getAttribute('cols').item()
+        fixed = entry.getAttribute('fixed').item()
+        label = int(entry.getAttribute(empty_entry_label, 'label').item()) + 1
+
+        if fixed == 1 and label > -1:
+            fixedSud[row][col] = label
+
+            if prediction[row][col] != label:
+                print("Prediction fixed wrong at %i:%i is %i should be %i"%(row,col,prediction[row][col],label))            
+            
+
+    print("fixedSud:\n %s"%(fixedSud))
+
+    for i in range(6):
+        if len(prediction[i][:].unique()) != 6:
+            print("Prediction wrong at row %i - %s"%(i,prediction[i][:]))  
+        
+        if len(prediction[:][i].unique()) != 6:
+            print("Prediction wrong at col %i - %s"%(i,prediction[:][i]))
+        
+        # r, c = divmod(i, 3)   
+        # r *=3
+        # c *=3
+        # rIndices = torch.tensor([r, r+1, r+2])
+        # cIndices = torch.tensor([c, c+1, c+2])
+
+        # currentTable = torch.index_select(prediction, 0, rIndices)
+        # currentTable = torch.index_select(currentTable, 1, cIndices)
+        # #print("currentTable:\n %s"%(currentTable))
+
+        # currentTable = torch.reshape(currentTable, (1,6))
+        # if len(currentTable.unique()) != 6:
+        #     print("Prediction wrong at table %i - %s"%(i,currentTable))
+    
+for datanode in program1.populate(trainreader):
+    entries = datanode.getChildDataNodes(conceptName=empty_entry)
+    
+    testSudokuPrediction(entries)
+    
+    _sud = list(trainreader)[0]['sudoku']
+    
+    # for entry in entries:
+    #     # t = entry.getAttribute(empty_entry_label, 'ILP')
+    #     row = entry.getAttribute('rows').item()
+    #     col = entry.getAttribute('cols').item()
+    #     val = entry.getAttribute(empty_entry_label, 'ILP').argmax(dim=-1).item() + 1
+      #     assert val == _sud[row][col]
+        # print(t)
+        # predicted = (t == 1).nonzero(as_tuple=True)[0].item() + 1
+        # if entry.getAttribute('fixed').item() == 1:
+        #     assert entry.getAttribute('val').item() == predicted
+    break
+    
+# program.train(trainreader, train_epoch_num=150, c_warmup_iters=0, 
+#               Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')
+
+# program1.train(trainreader, train_epoch_num=100, 
+#                     Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')
+
+trainingNo = 220
+for i in range(trainingNo):
+    print("Training - %i"%(i))
+    
+    program.train(trainreader, train_epoch_num=1,  c_warmup_iters=0,
+                    Optim=lambda param: torch.optim.Adam(param, lr=0.05), collectLoss=None, device='auto')
+    check = False
+    if program.model.loss.value()['empty_entry_label'].item() == 0:
+        print("loss is zero")
+        check = True
+    for datanode in program.populate(trainreader):
+        count = 0
+        _sud = list(trainreader)[0]['sudoku']
+        entries = datanode.getChildDataNodes(conceptName=empty_entry)
+        row_values = []
+        col_values = []
+        tab_values = []
+        for i in range(6):
+            row_values.append([])
+            col_values.append([])
+            tab_values.append([])
+        for entry in entries:
+            row = entry.getAttribute('rows').item()
+            col = entry.getAttribute('cols').item()
+            tab = entry.getAttribute('tables').item()
+            val = entry.getAttribute(empty_entry_label, 'local/argmax').argmax(dim=-1).item() + 1
+            row_values[row].append(val)
+            col_values[col].append(val)
+            tab_values[tab].append(val)
+            if check:
+                print("checking fixed stuff", entry.getAttribute('fixed').item())
+                if entry.getAttribute('fixed').item() == 1:
+                    print(entry.getAttribute('val').item(), val)
+                    assert entry.getAttribute('val').item() == val
+                    
+            if val != _sud[row][col]:
+                count += 1
+        
+        errors = 0
+        for row_vals in row_values:
+            un = set(row_vals)
+            errors += 6 - len(un)
+        for col_vals in col_values:
+            un = set(col_vals)
+            errors += 6 - len(un)
+        for tab_vals in tab_values:
+            un = set(tab_vals)
+            errors += 6 - len(un)
+                
+        print("Count of sudoku entries different from label - %s"%(count))
+        print("Count of sudoku violations - %s"%(errors))
+        
+
+### make the table
+for datanode in program.populate(trainreader):
+    datanode.inferILPResults(empty_entry_label, fun=None)
+
+    print("Datanode %s"%(datanode))
+    table = torch.zeros(6, 6)
+    ilpTable = torch.zeros(6, 6)
+
+    entries = datanode.getChildDataNodes(conceptName=empty_entry)
+    for entry in entries:
+        t = entry.getAttribute(empty_entry_label, 'local/argmax')
+        predicted = (t == 1).nonzero(as_tuple=True)[0].item() + 1
+        table[entry.getAttribute('rows').item()][entry.getAttribute('cols').item()] = predicted
+        
+        ilpPredicted = entry.getAttribute(empty_entry_label, 'ILP').argmax(dim=-1).item() + 1
+        ilpTable[entry.getAttribute('rows').item()][entry.getAttribute('cols').item()] = ilpPredicted
+
+    print("Argmax Predicted Table")
+    testSudokuPrediction(entries, predictionP = table)
+    
+    print("\n ILP Predicted Table")
+    testSudokuPrediction(entries, predictionP = ilpTable)
+    break
+    
diff --git a/examples/sudoku/main_simple.ipynb b/examples/sudoku/main_simple.ipynb
index f0b06398c..b4fb95239 100644
--- a/examples/sudoku/main_simple.ipynb
+++ b/examples/sudoku/main_simple.ipynb
@@ -3,15 +3,15 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "id": "several-baking",
+   "id": "amino-newfoundland",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/home/hfaghihi/Framework/new/DomiKnowS/examples\n",
-      "root Folder Absoloute path:  /home/hfaghihi/Framework/new/DomiKnowS\n"
+      "/home/hfaghihi/Framework/fix/DomiKnowS/examples\n",
+      "root Folder Absoloute path:  /home/hfaghihi/Framework/fix/DomiKnowS\n"
      ]
     }
    ],
@@ -43,7 +43,7 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "id": "tired-seeking",
+   "id": "clean-melbourne",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -91,45 +91,45 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "id": "comprehensive-wells",
+   "id": "exact-proceeding",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'id': [1], 'size': (9, 9), 'whole_sudoku': tensor([[0, 0, 0, 7, 3, 0, 0, 0, 0],\n",
-      "        [8, 4, 0, 0, 0, 0, 0, 0, 0],\n",
-      "        [0, 0, 0, 0, 5, 0, 0, 1, 0],\n",
-      "        [3, 0, 0, 8, 0, 0, 0, 7, 0],\n",
-      "        [0, 0, 0, 0, 0, 0, 0, 9, 8],\n",
-      "        [0, 0, 0, 0, 0, 1, 0, 4, 0],\n",
-      "        [0, 0, 6, 0, 0, 4, 0, 0, 0],\n",
-      "        [0, 0, 0, 0, 0, 7, 4, 0, 3],\n",
-      "        [4, 0, 0, 0, 0, 0, 7, 0, 1]])}\n",
-      "tensor([[0, 3],\n",
+      "{'id': [1], 'size': (9, 9), 'whole_sudoku': tensor([[0, 6, 1, 0, 8, 5, 0, 0, 3],\n",
+      "        [0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
+      "        [0, 8, 0, 0, 3, 0, 0, 0, 6],\n",
+      "        [0, 0, 3, 0, 0, 0, 0, 8, 0],\n",
+      "        [0, 0, 0, 0, 0, 0, 1, 0, 0],\n",
+      "        [0, 0, 0, 0, 0, 0, 7, 0, 0],\n",
+      "        [8, 0, 0, 0, 0, 3, 0, 9, 7],\n",
+      "        [0, 0, 9, 0, 0, 0, 3, 0, 0],\n",
+      "        [3, 0, 0, 0, 0, 0, 8, 0, 0]])}\n",
+      "tensor([[0, 1],\n",
+      "        [0, 2],\n",
       "        [0, 4],\n",
-      "        [1, 0],\n",
-      "        [1, 1],\n",
+      "        [0, 5],\n",
+      "        [0, 8],\n",
+      "        [1, 3],\n",
+      "        [2, 1],\n",
       "        [2, 4],\n",
-      "        [2, 7],\n",
-      "        [3, 0],\n",
-      "        [3, 3],\n",
+      "        [2, 8],\n",
+      "        [3, 2],\n",
       "        [3, 7],\n",
-      "        [4, 7],\n",
-      "        [4, 8],\n",
-      "        [5, 5],\n",
-      "        [5, 7],\n",
-      "        [6, 2],\n",
+      "        [4, 6],\n",
+      "        [5, 6],\n",
+      "        [6, 0],\n",
       "        [6, 5],\n",
-      "        [7, 5],\n",
+      "        [6, 7],\n",
+      "        [6, 8],\n",
+      "        [7, 2],\n",
       "        [7, 6],\n",
-      "        [7, 8],\n",
       "        [8, 0],\n",
-      "        [8, 6],\n",
-      "        [8, 8]])\n",
-      "(tensor([0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8]), tensor([3, 4, 0, 1, 4, 7, 0, 3, 7, 7, 8, 5, 7, 2, 5, 5, 6, 8, 0, 6, 8]))\n",
-      "tensor([7, 3, 8, 4, 5, 1, 3, 8, 7, 9, 8, 1, 4, 6, 4, 7, 4, 3, 4, 7, 1])\n"
+      "        [8, 6]])\n",
+      "(tensor([0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 6, 6, 7, 7, 8, 8]), tensor([1, 2, 4, 5, 8, 3, 1, 4, 8, 2, 7, 6, 6, 0, 5, 7, 8, 2, 6, 0, 6]))\n",
+      "tensor([6, 1, 8, 5, 3, 1, 8, 3, 6, 3, 8, 1, 7, 8, 3, 9, 7, 9, 3, 3, 8])\n"
      ]
     }
    ],
@@ -146,15 +146,14 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "higher-aquarium",
+   "id": "usual-noise",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Log file for dataNode is in: /home/hfaghihi/Framework/new/DomiKnowS/examples/sudoku/logs/datanode.log\n",
-      "col1\n"
+      "Log file for dataNode is in: /home/hfaghihi/Framework/fix/DomiKnowS/examples/sudoku/logs/datanode.log\n"
      ]
     },
     {
@@ -202,7 +201,7 @@
     "    same_col = Concept(name=\"same_col\")\n",
     "#     same_col_mixed = Concept(name=\"same_col_mixed\")\n",
     "    (same_col_arg1, same_col_arg2) = same_col.has_a(col1=empty_entry, col2=empty_entry)\n",
-    "    print(same_col_arg1)\n",
+    "#     print(same_col_arg1)\n",
     "#     (same_col_mixed_arg1, same_col_mixed_arg2) = same_col_mixed.has_a(arg1=empty_entry, arg2=fixed_entry)\n",
     "\n",
     "    same_table = Concept(name=\"same_table\")\n",
@@ -217,15 +216,34 @@
     "    # entry_label= entry(name=\"label\")\n",
     "    # fixedL(entry_label(\"x\", eqL(entry, \"given\", {True})))\n",
     "    \n",
-    "    fixedL(empty_entry_label(\"x\", eqL(empty_entry, \"fixed\", {True})))\n",
+    "#     fixedL(empty_entry_label(\"x\", eqL(empty_entry, \"fixed\", {True})))\n",
     "    \n",
     "    #fixedL(empty_entry_label(\"x\", path=('x', eqL(empty_entry, \"fixed\", {True}))))\n",
-    "\n",
-    "    exactL(getattr(empty_entry_label, f'v1'), getattr(empty_entry_label, f'v2'), getattr(empty_entry_label, f'v3'),\n",
-    "          getattr(empty_entry_label, f'v4'), getattr(empty_entry_label, f'v5'), getattr(empty_entry_label, f'v6'), getattr(empty_entry_label, f'v7'),\n",
-    "          getattr(empty_entry_label, f'v8'), getattr(empty_entry_label, f'v9'))\n",
+    "    ifL(empty_entry, \n",
+    "        exactL(getattr(empty_entry_label, f'v1'), getattr(empty_entry_label, f'v2'), getattr(empty_entry_label, f'v3'),\n",
+    "              getattr(empty_entry_label, f'v4'), getattr(empty_entry_label, f'v5'), getattr(empty_entry_label, f'v6'), getattr(empty_entry_label, f'v7'),\n",
+    "              getattr(empty_entry_label, f'v8'), getattr(empty_entry_label, f'v9'))\n",
+    "    )\n",
+    "    \n",
+    "    FIXED = True\n",
+    "    \n",
+    "    existsL_LCs = True\n",
+    "    atMostL_LCs = False\n",
+    "    \n",
+    "    SAME_ROW = existsL_LCs\n",
+    "    SAME_ROW_New = atMostL_LCs\n",
+    "    \n",
+    "    SAME_COLUMNE = existsL_LCs\n",
+    "    SAME_COLUMNE_New = atMostL_LCs\n",
     "    \n",
-    "    for val in [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\"]:\n",
+    "    SAME_TABLE = existsL_LCs\n",
+    "    SAME_TABLE_New = atMostL_LCs\n",
+    "    \n",
+    "    fixedL(empty_entry_label(\"x\", eqL(empty_entry, \"fixed\", {True})), active = FIXED)\n",
+    "    \n",
+    "    #fixedL(empty_entry_label(\"x\", path=('x', eqL(empty_entry, \"fixed\", {True}))))\n",
+    "    \n",
+    "    for val in [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\"]:        \n",
     "        ### No same number in the same row between empty entries and empty entries\n",
     "        ifL(getattr(empty_entry_label, f'v{val}')('x'), \n",
     "            notL(\n",
@@ -234,7 +252,16 @@
     "                        same_row('z', path=(\"x\", same_row_arg1.reversed)), \n",
     "                        getattr(empty_entry_label, f'v{val}')('y', path=(\"z\", same_row_arg2))\n",
     "                ))\n",
-    "        ))\n",
+    "        ), active = SAME_ROW, name = \"LC_SAME_ROW_ExistL_for_\" + f'v{val}')\n",
+    "        \n",
+    "        ifL(getattr(empty_entry_label, f'v{val}')('x'), \n",
+    "            atMostL(\n",
+    "                andL(\n",
+    "                    same_row('z', path=(\"x\", same_row_arg1.reversed)), \n",
+    "                    getattr(empty_entry_label, f'v{val}')('y', path=(\"z\", same_row_arg2)\n",
+    "                )\n",
+    "            )\n",
+    "        ), active = SAME_ROW_New, name = \"LC_SAME_ROW_atMostL_for_\" + f'v{val}')\n",
     "        \n",
     "        ### No same number in the same column between empty entries and empty entries\n",
     "        ifL(getattr(empty_entry_label, f'v{val}')('x'), \n",
@@ -244,7 +271,16 @@
     "                        same_col('z', path=(\"x\", same_col_arg1.reversed)), \n",
     "                        getattr(empty_entry_label, f'v{val}')('y', path=(\"z\", same_col_arg2))\n",
     "                ))\n",
-    "        ))\n",
+    "        ), active = SAME_COLUMNE, name = \"LC_SAME_COLUMNE_existsL_for_\" + f'v{val}')\n",
+    "        \n",
+    "        ifL(getattr(empty_entry_label, f'v{val}')('x'), \n",
+    "            atMostL(\n",
+    "                andL(\n",
+    "                    same_col('z', path=(\"x\", same_col_arg1.reversed)), \n",
+    "                    getattr(empty_entry_label, f'v{val}')('y', path=(\"z\", same_col_arg2)\n",
+    "                )\n",
+    "            )\n",
+    "        ), active = SAME_COLUMNE_New, name = \"LC_SAME_COLUMNE_atMostL_for_\" + f'v{val}')\n",
     "        \n",
     "        ### No same number in the same table between empty entries and empty entries\n",
     "        ifL(getattr(empty_entry_label, f'v{val}')('x'), \n",
@@ -254,13 +290,22 @@
     "                        same_table('z', path=(\"x\", same_table_arg1.reversed)), \n",
     "                        getattr(empty_entry_label, f'v{val}')('y', path=(\"z\", same_table_arg2))\n",
     "                ))\n",
-    "        ))\n"
+    "        ), active = SAME_TABLE, name = \"LC_SAME_TABLE_existsL_for_\" + f'v{val}')\n",
+    "        \n",
+    "        ifL(getattr(empty_entry_label, f'v{val}')('x'), \n",
+    "            atMostL(\n",
+    "                    andL(\n",
+    "                        same_table('z', path=(\"x\", same_table_arg1.reversed)), \n",
+    "                        getattr(empty_entry_label, f'v{val}')('y', path=(\"z\", same_table_arg2)\n",
+    "                    )\n",
+    "            )\n",
+    "        ), active = SAME_TABLE_New, name = \"LC_SAME_TABLE_atMostL_for_\" + f'v{val}')\n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "traditional-illinois",
+   "id": "dominant-suggestion",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -298,9 +343,30 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "chronic-vacation",
+   "id": "seeing-ultimate",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "CUDA error: out of memory\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/examples/sudoku/main.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0mempty_entry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fixed'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'val'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mJointFunctionalReaderSensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'rows'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'cols'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mempty_rel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeyword\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'whole_sudoku'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgetfixed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m \u001b[0mempty_entry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mempty_entry_label\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModuleLearner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'val'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSudokuSolver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m \u001b[0mempty_entry\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mempty_entry_label\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFunctionalReaderSensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeyword\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'whole_sudoku'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgetlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/learners.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, module, edges, loss, metric, label, *pres, **kwargs)\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mModuleLearner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModuleSensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTorchLearner\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mpres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0medges\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0medges\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0medges\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_metric\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, module, *args, **kwargs)\u001b[0m\n\u001b[1;32m    435\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    436\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 437\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    439\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/learners.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, edges, loss, metric, label, device, *pre)\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mpre\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0medges\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'auto'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdated\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTorchLearner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpre\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0medges\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0medges\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_metric\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, forward, build, *pres, **kwargs)\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mFunctionalSensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTorchSensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mpres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, edges, label, device, *pres)\u001b[0m\n\u001b[1;32m     20\u001b[0m             \u001b[0mis_cuda\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_cuda\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"cuda\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"cpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36mdevice\u001b[0;34m(self, device)\u001b[0m\n\u001b[1;32m    447\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_device\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mto\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    850\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_complex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 852\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    853\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m     def register_backward_hook(\n",
+      "\u001b[0;32m~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    550\u001b[0m                 \u001b[0;31m# `with torch.no_grad():`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 552\u001b[0;31m                     \u001b[0mparam_applied\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    553\u001b[0m                 \u001b[0mshould_use_set_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    554\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mshould_use_set_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    848\u001b[0m                 return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None,\n\u001b[1;32m    849\u001b[0m                             non_blocking, memory_format=convert_to_format)\n\u001b[0;32m--> 850\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_complex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    852\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: out of memory\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1."
+     ]
+    }
+   ],
    "source": [
     "from regr.sensor.pytorch.sensors import FunctionalSensor, JointSensor, ReaderSensor, FunctionalReaderSensor\n",
     "from regr.sensor.pytorch.learners import ModuleLearner\n",
@@ -398,8 +464,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "id": "happy-withdrawal",
+   "execution_count": 9,
+   "id": "generous-detection",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -411,7 +477,7 @@
     "from regr.program.model.pytorch import SolverModel\n",
     "\n",
     "program1 = SolverPOIProgram(\n",
-    "        graph, poi=(sudoku, empty_entry, same_row, same_col, same_table), inferTypes=['local/argmax', \"ILP\"],\n",
+    "        graph, poi=(sudoku, empty_entry, same_row, same_col, same_table), inferTypes=['local/argmax'],\n",
     "        loss=MacroAverageTracker(NBCrossEntropyLoss()),\n",
     "#         metric={\n",
     "#             'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))}\n",
@@ -433,14 +499,14 @@
     "        \n",
     "        sample = True,\n",
     "        sampleSize=300, \n",
-    "        sampleGlobalLoss = True\n",
+    "        sampleGlobalLoss = False\n",
     "        )"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "broad-click",
+   "execution_count": 25,
+   "id": "horizontal-plaintiff",
    "metadata": {},
    "outputs": [
     {
@@ -449,39 +515,36 @@
      "text": [
       "INFO:regr.program.program:Epoch: 1\n",
       "INFO:regr.program.program:Training:\n",
-      "Epoch 1 Training:   0%|          | 0/1 [00:00<?, ?it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Log file for ilpOntSolver is in: /home/hfaghihi/Framework/new/DomiKnowS/examples/sudoku/logs/ilpOntSolver.log\n",
-      "Academic license - for non-commercial use only - expires 2022-09-15\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:gurobipy.gurobipy:Academic license - for non-commercial use only - expires 2022-09-15\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Using license file /home/hfaghihi/gurobi.lic\n"
+      "Epoch 1 Training: 100%|██████████| 1/1 [00:04<00:00,  4.40s/it]\n",
+      "INFO:regr.program.program: - loss:\n",
+      "INFO:regr.program.program:{'empty_entry_label': tensor(1.7064)}\n",
+      "INFO:regr.program.program:Epoch: 2\n",
+      "INFO:regr.program.program:Training:\n",
+      "Epoch 2 Training:   0%|          | 0/1 [00:02<?, ?it/s]\n"
      ]
     },
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:gurobipy.gurobipy:Using license file /home/hfaghihi/gurobi.lic\n",
-      "Epoch 1 Training: 100%|██████████| 1/1 [00:24<00:00, 24.08s/it]\n",
-      "INFO:regr.program.program: - loss:\n",
-      "INFO:regr.program.program:{'empty_entry_label': tensor(2.2917)}\n"
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/examples/sudoku/main.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# with io.capture_output() as captured:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m program1.train(trainreader, train_epoch_num=100,\n\u001b[0;32m----> 5\u001b[0;31m               Optim=lambda param: torch.optim.SGD(param, lr=0.1), device='auto')\n\u001b[0m",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/program.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, training_set, valid_set, test_set, device, train_epoch_num, test_every_epoch, Optim, **kwargs)\u001b[0m\n\u001b[1;32m    268\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Epoch: %d'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 270\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    271\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Validation'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    272\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtest_every_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/program.py\u001b[0m in \u001b[0;36mcall_epoch\u001b[0;34m(self, name, dataset, epoch_fn, **kwargs)\u001b[0m\n\u001b[1;32m    207\u001b[0m             \u001b[0mdesc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34mf'Epoch {self.epoch} {name}'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m             \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_len\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdesc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/utils.py\u001b[0m in \u001b[0;36mconsume\u001b[0;34m(it)\u001b[0m\n\u001b[1;32m    297\u001b[0m     \u001b[0;32mimport\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 299\u001b[0;31m         \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeque\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    300\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m                 \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/program.py\u001b[0m in \u001b[0;36mtrain_epoch\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m    284\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    285\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 286\u001b[0;31m             \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_item\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    287\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    288\u001b[0m                 \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/model/pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, data_item, build)\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneedsBatchRootDN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                 \u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddBatchRootDN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m             \u001b[0;34m*\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m             \u001b[0mdatanode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetDataNode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatanode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/model/pytorch.py\u001b[0m in \u001b[0;36mpopulate\u001b[0;34m(self, builder, run)\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    245\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuilder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 246\u001b[0;31m         \u001b[0mdata_item\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minference\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    247\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/model/pytorch.py\u001b[0m in \u001b[0;36minference\u001b[0;34m(self, builder)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mprop\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0msensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTorchSensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m                 \u001b[0msensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;31m#             for output_sensor, target_sensor in self.find_sensors(prop):\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m \u001b[0;31m#             # make sure the sensors are evaluated\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, data_item)\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext_helper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_item\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_item\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Error {} during updating data item {} with sensor {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_item\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfullname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36mupdate_context\u001b[0;34m(self, data_item, force, override)\u001b[0m\n\u001b[1;32m    158\u001b[0m             \u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_wrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m             \u001b[0mdata_item\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprop\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_item\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0moverride\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m             \u001b[0mdata_item\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprop\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval\u001b[0m  \u001b[0;31m# override state under property name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/graph/dataNode.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, _key, value)\u001b[0m\n\u001b[1;32m   2138\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mconceptInfo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'relation'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2139\u001b[0m                 \u001b[0m_DataNodeBulder__Logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%s is a relation'\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_conceptName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2140\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__buildRelationLink\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvInfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconceptInfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeyDataName\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Build or update relation link\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2142\u001b[0m         \u001b[0;31m# Add value to the underling dictionary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/graph/dataNode.py\u001b[0m in \u001b[0;36m__buildRelationLink\u001b[0;34m(self, vInfo, conceptInfo, keyDataName)\u001b[0m\n\u001b[1;32m   1663\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1664\u001b[0m                         \u001b[0;32mfor\u001b[0m \u001b[0mcandidateIndex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcandidate\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcandidatesForRelation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1665\u001b[0;31m                             \u001b[0misInRelation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcandidate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1666\u001b[0m                             \u001b[0;32mif\u001b[0m \u001b[0misInRelation\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1667\u001b[0m                                 \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
      ]
     }
    ],
@@ -489,21 +552,21 @@
     "from IPython.utils import io\n",
     "\n",
     "# with io.capture_output() as captured:\n",
-    "program1.train(trainreader, train_epoch_num=1,\n",
+    "program1.train(trainreader, train_epoch_num=100,\n",
     "              Optim=lambda param: torch.optim.SGD(param, lr=0.1), device='auto')"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 9,
-   "id": "labeled-outdoors",
+   "id": "monetary-champagne",
    "metadata": {},
    "outputs": [],
    "source": [
     "data = None\n",
     "for datanode in program1.populate(trainreader):\n",
     "#     print(datanode)\n",
-    "    datanode.inferILPResults(empty_entry_label, same_row, same_col, same_table, fun=None)\n",
+    "#     datanode.inferILPResults(empty_entry_label, same_row, same_col, same_table, fun=None)\n",
     "    data = datanode\n",
     "    entries = datanode.getChildDataNodes(conceptName=empty_entry)\n",
     "    table = torch.zeros(9, 9)\n",
@@ -517,6 +580,8 @@
     "#         print(t)\n",
     "#         print(entry.getAttribute('fixed').item())\n",
     "#         print((t == 1).nonzero(as_tuple=True)[0].item() + 1)\n",
+    "#         print((t == 1))\n",
+    "        assert len((t == 1).nonzero(as_tuple=True)) == 1\n",
     "        predicted = (t == 1).nonzero(as_tuple=True)[0].item() + 1\n",
     "        table[entry.getAttribute('rows').item()][entry.getAttribute('cols').item()] = predicted\n",
     "        if entry.getAttribute('fixed').item() == 1:\n",
@@ -527,8 +592,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
-   "id": "excessive-athletics",
+   "execution_count": 10,
+   "id": "chief-creation",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -553,25 +618,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
-   "id": "eight-procurement",
+   "execution_count": 11,
+   "id": "increased-beverage",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "tensor([[5., 9., 4., 7., 5., 4., 8., 7., 3.],\n",
-       "        [5., 5., 6., 7., 5., 1., 7., 3., 8.],\n",
-       "        [9., 6., 4., 1., 4., 2., 1., 7., 5.],\n",
-       "        [1., 3., 9., 5., 8., 8., 4., 1., 8.],\n",
-       "        [6., 6., 1., 7., 4., 4., 3., 9., 9.],\n",
-       "        [5., 2., 5., 5., 5., 9., 3., 7., 3.],\n",
-       "        [2., 2., 7., 4., 4., 1., 5., 9., 4.],\n",
-       "        [7., 1., 6., 1., 6., 8., 8., 8., 4.],\n",
-       "        [3., 5., 6., 3., 3., 8., 1., 6., 4.]])"
+       "tensor([[5., 2., 4., 9., 8., 7., 1., 3., 6.],\n",
+       "        [8., 6., 3., 1., 5., 2., 9., 7., 4.],\n",
+       "        [1., 7., 9., 3., 6., 4., 5., 8., 2.],\n",
+       "        [2., 9., 7., 8., 4., 5., 6., 1., 3.],\n",
+       "        [3., 1., 6., 7., 2., 9., 4., 5., 8.],\n",
+       "        [4., 5., 8., 6., 3., 1., 2., 9., 7.],\n",
+       "        [9., 3., 2., 5., 7., 6., 8., 4., 1.],\n",
+       "        [7., 4., 5., 2., 1., 8., 3., 6., 9.],\n",
+       "        [6., 8., 1., 4., 9., 3., 7., 2., 5.]])"
       ]
      },
-     "execution_count": 12,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -582,8 +647,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "id": "strong-cisco",
+   "execution_count": 26,
+   "id": "asian-rebate",
    "metadata": {},
    "outputs": [
     {
@@ -592,13 +657,14 @@
      "text": [
       "INFO:regr.program.program:Epoch: 1\n",
       "INFO:regr.program.program:Training:\n",
-      "Epoch 1 Training: 100%|██████████| 1/1 [00:18<00:00, 18.21s/it]\n",
+      "Epoch 1 Training: 100%|██████████| 1/1 [00:09<00:00,  9.83s/it]\n",
+      "INFO:regr.program.program: - loss:\n",
+      "INFO:regr.program.program:None\n",
+      "INFO:regr.program.program: - Constraint loss:\n",
+      "INFO:regr.program.program:{'LC1': tensor(5.5993), 'LC10': tensor(39.1760), 'LC17': tensor(39.1433), 'LC24': tensor(39.5451), 'LC31': tensor(40.3391), 'LC38': tensor(39.9354), 'LC45': tensor(39.7444), 'LC52': tensor(43.3714), 'LC59': tensor(43.6981), 'LC66': tensor(43.5512), 'LC73': tensor(39.5709), 'LC80': tensor(40.2105), 'LC87': tensor(39.8340), 'LC94': tensor(43.6860), 'LC101': tensor(43.2489), 'LC108': tensor(43.9003), 'LC115': tensor(42.3305), 'LC122': tensor(42.6638), 'LC129': tensor(42.2624), 'LC136': tensor(43.9275), 'LC143': tensor(44.5062), 'LC150': tensor(43.8287), 'LC157': tensor(45.0451), 'LC164': tensor(45.2656), 'LC171': tensor(44.8230), 'LC178': tensor(42.3420), 'LC185': tensor(41.9413), 'LC192': tensor(42.1943)}\n",
       "INFO:regr.program.program:Epoch: 2\n",
       "INFO:regr.program.program:Training:\n",
-      "Epoch 2 Training: 100%|██████████| 1/1 [00:18<00:00, 18.30s/it]\n",
-      "INFO:regr.program.program:Epoch: 3\n",
-      "INFO:regr.program.program:Training:\n",
-      "Epoch 3 Training:   0%|          | 0/1 [00:11<?, ?it/s]\n"
+      "Epoch 2 Training:   0%|          | 0/1 [00:02<?, ?it/s]\n"
      ]
     },
     {
@@ -608,34 +674,62 @@
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/examples/sudoku/main.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# with io.capture_output() as captured:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m program.train(trainreader, train_epoch_num=50, c_warmup_iters=0, \n\u001b[0;32m----> 5\u001b[0;31m               Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/program/lossprogram.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, training_set, valid_set, test_set, c_lr, c_momentum, c_warmup_iters, c_freq, c_freq_increase, c_freq_increase_freq, c_lr_decay, c_lr_decay_param, **kwargs)\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0mc_lr_decay_param\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mc_lr_decay_param\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m             \u001b[0mc_session\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mc_session\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m             **kwargs)\n\u001b[0m\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m     def train_epoch(\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/program/program.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, training_set, valid_set, test_set, device, train_epoch_num, test_every_epoch, Optim, **kwargs)\u001b[0m\n\u001b[1;32m    268\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Epoch: %d'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 270\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    271\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Validation'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    272\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtest_every_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/program/program.py\u001b[0m in \u001b[0;36mcall_epoch\u001b[0;34m(self, name, dataset, epoch_fn, **kwargs)\u001b[0m\n\u001b[1;32m    207\u001b[0m             \u001b[0mdesc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34mf'Epoch {self.epoch} {name}'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m             \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_len\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdesc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/utils.py\u001b[0m in \u001b[0;36mconsume\u001b[0;34m(it)\u001b[0m\n\u001b[1;32m    297\u001b[0m     \u001b[0;32mimport\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 299\u001b[0;31m         \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeque\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    300\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/examples/sudoku/main.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# with io.capture_output() as captured:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m program.train(trainreader, train_epoch_num=150, c_warmup_iters=0, \n\u001b[0;32m----> 7\u001b[0;31m               Optim=lambda param: torch.optim.SGD(param, lr=0.1), device='auto')\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/lossprogram.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, training_set, valid_set, test_set, c_lr, c_momentum, c_warmup_iters, c_freq, c_freq_increase, c_freq_increase_freq, c_lr_decay, c_lr_decay_param, **kwargs)\u001b[0m\n\u001b[1;32m     95\u001b[0m             \u001b[0mc_lr_decay_param\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mc_lr_decay_param\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m             \u001b[0mc_session\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mc_session\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m             **kwargs)\n\u001b[0m\u001b[1;32m     98\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/program.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, training_set, valid_set, test_set, device, train_epoch_num, test_every_epoch, Optim, **kwargs)\u001b[0m\n\u001b[1;32m    268\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Epoch: %d'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 270\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    271\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Validation'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    272\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtest_every_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/lossprogram.py\u001b[0m in \u001b[0;36mcall_epoch\u001b[0;34m(self, name, dataset, epoch_fn, **kwargs)\u001b[0m\n\u001b[1;32m    102\u001b[0m             \u001b[0mdesc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34mf'Epoch {self.epoch} {name}'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_len\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdesc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/utils.py\u001b[0m in \u001b[0;36mconsume\u001b[0;34m(it)\u001b[0m\n\u001b[1;32m    297\u001b[0m     \u001b[0;32mimport\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 299\u001b[0;31m         \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeque\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    300\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m                 \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/program/lossprogram.py\u001b[0m in \u001b[0;36mtrain_epoch\u001b[0;34m(self, dataset, c_lr, c_warmup_iters, c_freq_increase, c_freq_increase_freq, c_lr_decay, c_lr_decay_param, c_session, **kwargs)\u001b[0m\n\u001b[1;32m    123\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m                 \u001b[0mcloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    126\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmloss\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbeta\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mcloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/lossprogram.py\u001b[0m in \u001b[0;36mtrain_epoch\u001b[0;34m(self, dataset, c_lr, c_warmup_iters, c_freq_increase, c_freq_increase_freq, c_lr_decay, c_lr_decay_param, c_session, **kwargs)\u001b[0m\n\u001b[1;32m    173\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 175\u001b[0;31m             \u001b[0mmloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# output = (datanode, builder)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    176\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0miter\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mc_warmup_iters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/program/model/lossModel.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, builder, build)\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0;31m# Call the loss calculation returns a dictionary, keys are matching the constraints\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m         \u001b[0mconstr_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatanode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcalculateLcLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtnorm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtnorm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msampleSize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msampleSize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0mlmbd_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/graph/dataNode.py\u001b[0m in \u001b[0;36mcalculateLcLoss\u001b[0;34m(self, tnorm, sample, sampleSize)\u001b[0m\n\u001b[1;32m   1134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minferLocal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1136\u001b[0;31m         \u001b[0mlcResult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmyilpOntSolver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcalculateLcLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtnorm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtnorm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msampleSize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msampleSize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlcResult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/Framework/new/DomiKnowS/regr/solver/gurobiILPOntSolver.py\u001b[0m in \u001b[0;36mcalculateLcLoss\u001b[0;34m(self, dn, tnorm, sample, sampleSize)\u001b[0m\n\u001b[1;32m   1178\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mlc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlcLosses\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m                     \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlcLosses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lossData'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m                         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m                             \u001b[0;31m#isGlobal = False\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m                             \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/model/pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, data_item, build)\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneedsBatchRootDN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                 \u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddBatchRootDN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m             \u001b[0;34m*\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m             \u001b[0mdatanode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetDataNode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatanode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/model/pytorch.py\u001b[0m in \u001b[0;36mpopulate\u001b[0;34m(self, builder, run)\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    245\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuilder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 246\u001b[0;31m         \u001b[0mdata_item\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minference\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    247\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/program/model/pytorch.py\u001b[0m in \u001b[0;36minference\u001b[0;34m(self, builder)\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mprop\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0msensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTorchSensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 221\u001b[0;31m                 \u001b[0msensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;31m#             for output_sensor, target_sensor in self.find_sensors(prop):\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m \u001b[0;31m#             # make sure the sensors are evaluated\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, data_item)\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext_helper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_item\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_item\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Error {} during updating data item {} with sensor {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_item\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfullname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/sensor/pytorch/sensors.py\u001b[0m in \u001b[0;36mupdate_context\u001b[0;34m(self, data_item, force, override)\u001b[0m\n\u001b[1;32m    158\u001b[0m             \u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_wrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m             \u001b[0mdata_item\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprop\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_item\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0moverride\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m             \u001b[0mdata_item\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprop\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval\u001b[0m  \u001b[0;31m# override state under property name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/graph/dataNode.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, _key, value)\u001b[0m\n\u001b[1;32m   2138\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mconceptInfo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'relation'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2139\u001b[0m                 \u001b[0m_DataNodeBulder__Logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%s is a relation'\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_conceptName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2140\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__buildRelationLink\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvInfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconceptInfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeyDataName\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Build or update relation link\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2142\u001b[0m         \u001b[0;31m# Add value to the underling dictionary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/graph/dataNode.py\u001b[0m in \u001b[0;36m__buildRelationLink\u001b[0;34m(self, vInfo, conceptInfo, keyDataName)\u001b[0m\n\u001b[1;32m   1686\u001b[0m                     \u001b[0mrDn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattributes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkeyDataName\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvInfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# Add / /Update value of the attribute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1688\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__updateRootDataNodeList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexistingDnsForRelationSorted\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1689\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1690\u001b[0m             \u001b[0;31m# -- DataNode with this relation already created  - update it with new attribute value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/graph/dataNode.py\u001b[0m in \u001b[0;36m__updateRootDataNodeList\u001b[0;34m(self, *dns)\u001b[0m\n\u001b[1;32m   1548\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__contains__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'dataNode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1549\u001b[0m             \u001b[0mdnsRoots\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'dataNode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1550\u001b[0;31m             \u001b[0m_DataNodeBulder__Logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Existing elements in the root dataNodes list - %s'\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdnsRoots\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1551\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1552\u001b[0m             \u001b[0mdnsRoots\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Framework/fix/DomiKnowS/regr/graph/dataNode.py\u001b[0m in \u001b[0;36m__repr__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minstanceValue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minstanceValue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
      ]
     }
    ],
    "source": [
     "from IPython.utils import io\n",
+    "from regr.utils import setProductionLogMode\n",
+    "setProductionLogMode()\n",
     "\n",
     "# with io.capture_output() as captured:\n",
-    "program.train(trainreader, train_epoch_num=50, c_warmup_iters=0, \n",
-    "              Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')\n",
+    "program.train(trainreader, train_epoch_num=150, c_warmup_iters=0, \n",
+    "              Optim=lambda param: torch.optim.SGD(param, lr=0.1), device='auto')\n",
     "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
-   "id": "bridal-seating",
+   "execution_count": 36,
+   "id": "altered-airplane",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "None\n",
+      "None\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(program.model.loss.value())\n",
+    "print(program1.model.loss)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "unlike-somerset",
    "metadata": {},
    "outputs": [
     {
@@ -645,327 +739,246 @@
       "sodoku 0\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(0, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(1, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "5\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(2, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "4\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "4\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "5\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(3, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "5\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "8\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(4, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "4\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "8\n",
       "---\n",
       "tensor(5, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "8\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(6, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "6\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "8\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "4\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "4\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "5\n",
       "---\n",
       "tensor(7, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(0, device='cuda:0')\n",
-      "9\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(1, device='cuda:0')\n",
-      "1\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(2, device='cuda:0')\n",
-      "7\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(3, device='cuda:0')\n",
-      "4\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(4, device='cuda:0')\n",
-      "2\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(5, device='cuda:0')\n",
-      "8\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(6, device='cuda:0')\n",
-      "3\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(7, device='cuda:0')\n",
-      "4\n",
       "---\n",
       "tensor(8, device='cuda:0')\n",
       "tensor(8, device='cuda:0')\n",
-      "4\n",
       "---\n"
      ]
     }
@@ -983,9 +996,11 @@
     "        t = entry.getAttribute(empty_entry_label, 'local/argmax')\n",
     "#         print((t == 1).nonzero(as_tuple=True)[0].item() + 1)\n",
     "        predicted = (t == 1).nonzero(as_tuple=True)[0].item() + 1\n",
+    "#         print(predicted)\n",
     "        table[entry.getAttribute('rows').item()][entry.getAttribute('cols').item()] = predicted\n",
-    "        print(predicted)\n",
+    "#         print(predicted)\n",
     "        if entry.getAttribute('fixed').item() == 1:\n",
+    "#             pass\n",
     "            assert entry.getAttribute('val').item() == predicted\n",
     "        print(\"---\")\n",
     "    break"
@@ -994,21 +1009,21 @@
   {
    "cell_type": "code",
    "execution_count": 18,
-   "id": "liquid-portrait",
+   "id": "marine-shelter",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "tensor([[2., 1., 9., 7., 1., 7., 2., 7., 7.],\n",
-       "        [3., 3., 3., 3., 7., 6., 3., 2., 6.],\n",
-       "        [5., 2., 9., 6., 1., 9., 7., 2., 4.],\n",
-       "        [6., 1., 4., 9., 3., 7., 5., 7., 7.],\n",
-       "        [6., 5., 6., 1., 2., 8., 9., 2., 9.],\n",
-       "        [4., 9., 9., 3., 9., 3., 1., 8., 8.],\n",
-       "        [6., 2., 1., 9., 3., 1., 6., 9., 9.],\n",
-       "        [6., 3., 3., 8., 3., 4., 4., 5., 3.],\n",
-       "        [9., 1., 7., 4., 2., 8., 3., 4., 4.]])"
+       "tensor([[9., 4., 9., 3., 3., 4., 9., 9., 2.],\n",
+       "        [2., 6., 3., 9., 8., 2., 9., 3., 1.],\n",
+       "        [3., 4., 6., 7., 7., 3., 1., 4., 6.],\n",
+       "        [2., 7., 3., 3., 1., 6., 2., 2., 2.],\n",
+       "        [5., 5., 2., 9., 8., 7., 1., 6., 5.],\n",
+       "        [5., 7., 5., 6., 8., 1., 9., 1., 6.],\n",
+       "        [4., 1., 7., 3., 8., 3., 2., 5., 8.],\n",
+       "        [3., 9., 4., 8., 6., 7., 1., 1., 8.],\n",
+       "        [4., 5., 8., 1., 2., 8., 6., 7., 3.]])"
       ]
      },
      "execution_count": 18,
@@ -1023,7 +1038,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "auburn-following",
+   "id": "proud-guard",
    "metadata": {},
    "outputs": [],
    "source": []
diff --git a/examples/sudoku/main_simple.py b/examples/sudoku/main_simple.py
index 3f982680c..8d41579b6 100644
--- a/examples/sudoku/main_simple.py
+++ b/examples/sudoku/main_simple.py
@@ -17,6 +17,9 @@
 from regr.program.model.pytorch import SolverModel
 from regr.utils import setProductionLogMode
 
+
+from random import sample
+
 class SudokuReader(RegrReader):
     def parse_file(self):
         base  = 3
@@ -35,27 +38,57 @@ def shuffle(s): return sample(s,len(s))
 
         # produce board using randomized baseline pattern
         board = [ [nums[pattern(r,c)] for c in cols] for r in rows ]
+        F = []
+        for i in board:
+            F.append([])
+            for j in i:
+                F[-1].append(j)
         squares = side*side
         empties = squares * 3//4
         for p in sample(range(squares),empties):
             board[p//side][p%side] = 0
         board = torch.tensor(board)
-    
-        return [{"board": board}]
+
+        board = torch.tensor([[6, 4, 0, 1, 0, 0, 8, 5, 9],
+                 [8, 9, 5, 6, 0, 4, 2, 7, 0],
+                 [0, 0, 0, 9, 0, 5, 0, 3, 4],
+                 [0, 3, 6, 0, 4, 8, 9, 2, 0],
+                 [9, 0, 2, 3, 0, 7, 0, 6, 5],
+                 [0, 7, 4, 0, 9, 6, 0, 8, 0],
+                 [4, 6, 0, 0, 2, 3, 5, 0, 8],
+                 [0, 0, 7, 0, 0, 9, 0, 4, 0],
+                 [3, 0, 8, 4, 0, 1, 0, 0, 0],
+                 ])
+
+        F = torch.tensor([[6, 4, 3, 1, 7, 2, 8, 5, 9],
+                 [8, 9, 5, 6, 3, 4, 2, 7, 1],
+                 [7, 2, 1, 9, 8, 5, 6, 3, 4],
+                 [1, 3, 6, 5, 4, 8, 9, 2, 7],
+                 [9, 8, 2, 3, 1, 7, 4, 6, 5],
+                 [5, 7, 4, 2, 9, 6, 1, 8, 3],
+                 [4, 6, 9, 7, 2, 3, 5, 1, 8],
+                 [2, 1, 7, 8, 5, 9, 3, 4, 6],
+                 [3, 5, 8, 4, 6, 1, 7, 9, 2],
+                 ])
+    
+        return [{"board": board, "F": F}]
     
     def getidval(self, item):
         return [1]
-    
     def getwhole_sudokuval(self, item):
         return item['board']
-            
+    
+    def getsudokuval(self, item):
+        return item['F']
+    
     def getsizeval(self, item):
         return 9, 9
     
+    
 trainreader = SudokuReader("randn", type="raw")
 
 from regr.graph import Graph, Concept, Relation
-from regr.graph.logicalConstrain import andL, existsL, notL, atMostL, ifL, fixedL, eqL
+from regr.graph.logicalConstrain import andL, existsL, notL, atMostL, ifL, fixedL, eqL, exactL
 from regr.graph import EnumConcept
 
 Graph.clear()
@@ -78,7 +111,7 @@ def getsizeval(self, item):
     
     same_row = Concept(name="same_row")
 #     same_row_mixed = Concept(name="same_row_mixed")
-    (same_row_arg1, same_row_arg2) = same_row.has_a(arg1=empty_entry, arg2=empty_entry)
+    (same_row_arg1, same_row_arg2) = same_row.has_a(row1=empty_entry, row2=empty_entry)
 #     (same_row_mixed_arg1, same_row_mixed_arg2) = same_row_mixed.has_a(arg1=empty_entry, arg2=fixed_entry)
     
     same_col = Concept(name="same_col")
@@ -88,10 +121,11 @@ def getsizeval(self, item):
 #     (same_col_mixed_arg1, same_col_mixed_arg2) = same_col_mixed.has_a(arg1=empty_entry, arg2=fixed_entry)
 
     same_table = Concept(name="same_table")
-    (same_table_arg1, same_table_arg2) = same_table.has_a(entry1=empty_entry, entry2=empty_entry)
+    (same_table_arg1, same_table_arg2) = same_table.has_a(table1=empty_entry, table2=empty_entry)
     
     empty_entry_label = empty_entry(name="empty_entry_label", ConceptClass=EnumConcept, 
                                     values=["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9"])
+    v = [getattr(empty_entry_label, a) for a in ('', *empty_entry_label.enum)]
 
     ### Constraints
     # entry = concept(name="entry")
@@ -101,80 +135,20 @@ def getsizeval(self, item):
     
     FIXED = True
     
-    existsL_LCs = False
-    atMostL_LCs = True
-    
-    SAME_ROW = existsL_LCs
-    SAME_ROW_New = atMostL_LCs
-    
-    SAME_COLUMNE = existsL_LCs
-    SAME_COLUMNE_New = atMostL_LCs
-    
-    SAME_TABLE = existsL_LCs
-    SAME_TABLE_New = atMostL_LCs
+    # ifL(empty_entry,  atMostL(*empty_entry_label.attributes), active = True, sampleEntries = True)
     
     fixedL(empty_entry_label("x", eqL(empty_entry, "fixed", {True})), active = FIXED)
     
-    #fixedL(empty_entry_label("x", path=('x', eqL(empty_entry, "fixed", {True}))))
-    
-    for val in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]:        
-        ### No same number in the same row between empty entries and empty entries
-        ifL(getattr(empty_entry_label, f'v{val}')('x'), 
-            notL(
-                existsL(
-                    andL(
-                        same_row('z', path=("x", same_row_arg1.reversed)), 
-                        getattr(empty_entry_label, f'v{val}')('y', path=("z", same_row_arg2))
-                ))
-        ), active = SAME_ROW, name = "LC_SAME_ROW_ExistL_for_" + f'v{val}')
-        
-        ifL(getattr(empty_entry_label, f'v{val}')('x'), 
-            atMostL(
-                andL(
-                    same_row('z', path=("x", same_row_arg1.reversed)), 
-                    getattr(empty_entry_label, f'v{val}')('y', path=("z", same_row_arg2)
-                )
-            )
-        ), active = SAME_ROW_New, name = "LC_SAME_ROW_atMostL_for_" + f'v{val}')
-        
-        ### No same number in the same column between empty entries and empty entries
-        ifL(getattr(empty_entry_label, f'v{val}')('x'), 
-            notL(
-                existsL(
-                    andL(
-                        same_col('z', path=("x", same_col_arg1.reversed)), 
-                        getattr(empty_entry_label, f'v{val}')('y', path=("z", same_col_arg2))
-                ))
-        ), active = SAME_COLUMNE, name = "LC_SAME_COLUMNE_existsL_for_" + f'v{val}')
-        
-        ifL(getattr(empty_entry_label, f'v{val}')('x'), 
-            atMostL(
-                andL(
-                    same_col('z', path=("x", same_col_arg1.reversed)), 
-                    getattr(empty_entry_label, f'v{val}')('y', path=("z", same_col_arg2)
-                )
-            )
-        ), active = SAME_COLUMNE_New, name = "LC_SAME_COLUMNE_atMostL_for_" + f'v{val}')
-        
-        ### No same number in the same table between empty entries and empty entries
-        ifL(getattr(empty_entry_label, f'v{val}')('x'), 
-            notL(
-                existsL(
-                    andL(
-                        same_table('z', path=("x", same_table_arg1.reversed)), 
-                        getattr(empty_entry_label, f'v{val}')('y', path=("z", same_table_arg2))
-                ))
-        ), active = SAME_TABLE, name = "LC_SAME_TABLE_existsL_for_" + f'v{val}')
-        
-        ifL(getattr(empty_entry_label, f'v{val}')('x'), 
-            atMostL(
-                    andL(
-                        same_table('z', path=("x", same_table_arg1.reversed)), 
-                        getattr(empty_entry_label, f'v{val}')('y', path=("z", same_table_arg2)
-                    )
-            )
-        ), active = SAME_TABLE_New, name = "LC_SAME_TABLE_atMostL_for_" + f'v{val}')
-        
+    for row_num in range(9):
+        andL(*[exactL(v[i](path = (eqL(empty_entry, "rows", {row_num})))) for i in range(1, 10)])
+        andL(*[exactL(v[i](path = (eqL(empty_entry, "cols", {row_num})))) for i in range(1, 10)])
+        andL(*[exactL(v[i](path = (eqL(empty_entry, "tables", {row_num})))) for i in range(1, 10)])
+
+        # for j in range(1, 10):
+        #     exactL(v[j](path = (eqL(empty_entry, "rows", {row_num}))))
+        #     exactL(v[j](path = (eqL(empty_entry, "cols", {row_num}))))
+        #     exactL(v[j](path = (eqL(empty_entry, "tables", {row_num}))))
+   
 import torch.nn as nn
 
 class Conv2dSame(torch.nn.Module):
@@ -220,7 +194,7 @@ def getfixed(*prev, data):
     for i, j in zip(rows.detach().tolist(), cols.detach().tolist()):
         fix[i][j] = 1
         vals[i][j] = data[i][j]
-        
+                
     return fix.reshape(fix.shape[0]*fix.shape[1]), vals.reshape(vals.shape[0]*vals.shape[1])
 
 def makeSoduko(*prev, data):
@@ -231,10 +205,20 @@ def makeSoduko(*prev, data):
     
     cols = torch.arange(num_rows)
     cols = cols.unsqueeze(0).repeat(num_rows, 1).reshape(num_rows*num_cols)
+
+    tables = []
+    for i in range(data[0]*data[1]):
+        row = rows[i]
+        col = cols[i]
+        x = int((col.item() / 3)) * 3
+        x += int(row.item() / 3) 
+        tables.append(x)
+    tables = torch.tensor(tables)
+
     
     rel = torch.ones(data[0]*data[1], 1)
     
-    return rows, cols, rel
+    return rows, cols, tables, rel
 
 def getlabel(*prev, data):
     rows, cols = torch.where(data != 0)
@@ -249,7 +233,7 @@ def createSudoku(*prev, data):
 
 sudoku['index'] = FunctionalReaderSensor(keyword='size', forward=createSudoku)
     
-empty_entry['rows', 'cols', empty_rel] = JointFunctionalReaderSensor(sudoku['index'], keyword='size', forward=makeSoduko)
+empty_entry['rows', 'cols', 'tables', empty_rel] = JointFunctionalReaderSensor(sudoku['index'], keyword='size', forward=makeSoduko)
 empty_entry['fixed', 'val'] = JointFunctionalReaderSensor('rows', 'cols', empty_rel, keyword='whole_sudoku', forward=getfixed)
 
 empty_entry[empty_entry_label] = ModuleLearner('val', module=SudokuSolver())
@@ -265,8 +249,8 @@ def filter_col(*inputs, col1, col2):
     relations=(same_col_arg1.reversed, same_col_arg2.reversed),
     forward=filter_col)
 
-def filter_row(*inputs, arg1, arg2):
-    if arg1.getAttribute('rows').item() == arg2.getAttribute('rows').item() and arg1.instanceID != arg2.instanceID:
+def filter_row(*inputs, row1, row2):
+    if row1.getAttribute('rows').item() == row2.getAttribute('rows').item() and row1.instanceID != row2.instanceID:
         return True
     return False
     
@@ -275,10 +259,10 @@ def filter_row(*inputs, arg1, arg2):
     relations=(same_row_arg1.reversed, same_row_arg2.reversed),
     forward=filter_row)
 
-def filter_table(*inputs, entry1, entry2):
-    if entry1.instanceID != entry2.instanceID:
-        if int(entry1.getAttribute('rows').item() / 3) == int(entry2.getAttribute('rows').item() / 3) and \
-           int(entry1.getAttribute('cols').item() / 3) == int(entry2.getAttribute('cols').item() / 3):
+def filter_table(*inputs, table1, table2):
+    if table1.instanceID != table2.instanceID:
+        if int(table1.getAttribute('rows').item() / 3) == int(table2.getAttribute('rows').item() / 3) and \
+           int(table1.getAttribute('cols').item() / 3) == int(table2.getAttribute('cols').item() / 3):
             return True
         
     return False
@@ -298,7 +282,7 @@ def filter_table(*inputs, entry1, entry2):
 from regr.program.loss import NBCrossEntropyLoss
 
 program1 = SolverPOIProgram(
-        graph, poi=(sudoku, empty_entry, same_row, same_col, same_table), inferTypes=['local/argmax', "ILP"],
+        graph, poi=(sudoku, empty_entry, same_row, same_col, same_table), inferTypes=['local/argmax', 'ILP'],
         loss=MacroAverageTracker(NBCrossEntropyLoss()),
 #         metric={
 #             'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))}
@@ -316,18 +300,20 @@ def filter_table(*inputs, entry1, entry2):
         #       'ILP': PRF1Tracker(DatanodeCMMetric()),
         #        'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))
         #       },
-#         loss=MacroAverageTracker(NBCrossEntropyLoss()),
+        loss=MacroAverageTracker(NBCrossEntropyLoss()),
         
         sample = True,
-        sampleSize=300, 
-        sampleGlobalLoss = True
+        sampleSize=2000, 
+        sampleGlobalLoss = False,
+        beta=1,
         )
 
-# program = SolverPOIProgram(
-#         graph, poi=(sudoku, empty_entry, ), inferTypes=['local/argmax'],
+# program1 = SolverPOIProgram(
+#         graph, poi=(sudoku, empty_entry), inferTypes=['local/argmax', 'ILP'],
 #         loss=MacroAverageTracker(NBCrossEntropyLoss()),
-#         metric={
-#             'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))})
+        # metric={
+        #     'argmax': PRF1Tracker(DatanodeCMMetric('local/argmax'))}
+            # )
 
 # program = SolverPOIProgram(
 #         graph, poi=(sudoku, empty_entry, same_col, same_row, same_table), inferTypes=['ILP', 'local/argmax'],
@@ -346,49 +332,170 @@ def filter_table(*inputs, entry1, entry2):
 #     print("sudokuLoss - %s"%(sudokuLoss))
 
 
-# Disable Logging    
-setProductionLogMode()
+# Disable Logging  
+productionMode = True  
+if productionMode:
+    setProductionLogMode()
 
-program1.train(trainreader, train_epoch_num=1, Optim=lambda param: torch.optim.SGD(param, lr=1), device='auto')
+# program1.train(trainreader, train_epoch_num=1, Optim=lambda param: torch.optim.SGD(param, lr=1), device='auto')
 
 ### test to see whether the FixedL is working, 
+def testSudokuPrediction(entries, predictionP = None):
+    
+    if predictionP != None:
+        prediction = predictionP
+    else:
+        prediction = torch.zeros((9,9))
+        for entry in entries:
+            row = entry.getAttribute('rows').item()
+            col = entry.getAttribute('cols').item()
+    
+            val = entry.getAttribute(empty_entry_label, 'ILP').argmax(dim=-1).item() + 1
+            prediction[row][col] = val
+          
+    print("Prediction:\n %s"%(prediction))
+     
+    fixedSud = torch.zeros((9,9)) #[[None for i in range(9)] for i in range(9)]
+    for entry in entries:
+        # t = entry.getAttribute(empty_entry_label, 'ILP')
+        row = entry.getAttribute('rows').item()
+        col = entry.getAttribute('cols').item()
+        fixed = entry.getAttribute('fixed').item()
+        label = int(entry.getAttribute(empty_entry_label, 'label').item()) + 1
 
-if  existsL_LCs:
-    print("\nUsing constraints with fixedL and existL")
-elif atMostL_LCs:
-    print("\nUsing constraints with fixedL and atMostL")
-else:
-    print("\nUsing only fixedL constraintsL")
+        if fixed == 1 and label > -1:
+            fixedSud[row][col] = label
+
+            if prediction[row][col] != label:
+                print("Prediction fixed wrong at %i:%i is %i should be %i"%(row,col,prediction[row][col],label))            
+            
+
+    print("fixedSud:\n %s"%(fixedSud))
+
+    for i in range(9):
+        if len(prediction[i][:].unique()) != 9:
+            print("Prediction wrong at row %i - %s"%(i,prediction[i][:]))  
+        
+        if len(prediction[:][i].unique()) != 9:
+            print("Prediction wrong at col %i - %s"%(i,prediction[:][i]))
+        
+        r, c = divmod(i, 3)   
+        r *=3
+        c *=3
+        rIndices = torch.tensor([r, r+1, r+2])
+        cIndices = torch.tensor([c, c+1, c+2])
+
+        currentTable = torch.index_select(prediction, 0, rIndices)
+        currentTable = torch.index_select(currentTable, 1, cIndices)
+        #print("currentTable:\n %s"%(currentTable))
+
+        currentTable = torch.reshape(currentTable, (1,9))
+        if len(currentTable.unique()) != 9:
+            print("Prediction wrong at table %i - %s"%(i,currentTable))
     
 for datanode in program1.populate(trainreader):
-    datanode.inferILPResults(empty_entry_label, fun=None)
     entries = datanode.getChildDataNodes(conceptName=empty_entry)
-    for entry in entries:
-        t = entry.getAttribute(empty_entry_label, 'ILP')
-        print(t)
-        predicted = (t == 1).nonzero(as_tuple=True)[0].item() + 1
-        if entry.getAttribute('fixed').item() == 1:
-            assert entry.getAttribute('val').item() == predicted
+    
+    testSudokuPrediction(entries)
+    
+    _sud = list(trainreader)[0]['sudoku']
+    
+    # for entry in entries:
+    #     # t = entry.getAttribute(empty_entry_label, 'ILP')
+    #     row = entry.getAttribute('rows').item()
+    #     col = entry.getAttribute('cols').item()
+    #     val = entry.getAttribute(empty_entry_label, 'ILP').argmax(dim=-1).item() + 1
+      #     assert val == _sud[row][col]
+        # print(t)
+        # predicted = (t == 1).nonzero(as_tuple=True)[0].item() + 1
+        # if entry.getAttribute('fixed').item() == 1:
+        #     assert entry.getAttribute('val').item() == predicted
     break
     
-program.train(trainreader, train_epoch_num=150, c_warmup_iters=0, 
-              Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')
+# program.train(trainreader, train_epoch_num=150, c_warmup_iters=0, 
+#               Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')
+
+# program1.train(trainreader, train_epoch_num=100, 
+#                     Optim=lambda param: torch.optim.SGD(param, lr=0.01), device='auto')
+
+trainingNo = 500
+for i in range(trainingNo):
+    print("Training - %i"%(i))
+    
+    program.train(trainreader, train_epoch_num=1,  c_warmup_iters=0,
+                    Optim=lambda param: torch.optim.Adam(param, lr=0.01), collectLoss=None, device='auto')
+    check = False
+    if program.model.loss.value()['empty_entry_label'].item() == 0:
+        print("loss is zero")
+        check = True
+    for datanode in program.populate(trainreader):
+        count = 0
+        _sud = list(trainreader)[0]['sudoku']
+        entries = datanode.getChildDataNodes(conceptName=empty_entry)
+        row_values = []
+        col_values = []
+        tab_values = []
+        for i in range(9):
+            row_values.append([])
+            col_values.append([])
+            tab_values.append([])
+        for entry in entries:
+            row = entry.getAttribute('rows').item()
+            col = entry.getAttribute('cols').item()
+            tab = entry.getAttribute('tables').item()
+            val = entry.getAttribute(empty_entry_label, 'local/argmax').argmax(dim=-1).item() + 1
+            row_values[row].append(val)
+            col_values[col].append(val)
+            tab_values[tab].append(val)
+            if check:
+                print("checking fixed stuff", entry.getAttribute('fixed').item())
+                if entry.getAttribute('fixed').item() == 1:
+                    print(entry.getAttribute('val').item(), val)
+                    assert entry.getAttribute('val').item() == val
+                    
+            if val != _sud[row][col]:
+                count += 1
+        
+        errors = 0
+        for row_vals in row_values:
+            un = set(row_vals)
+            errors += 9 - len(un)
+        for col_vals in col_values:
+            un = set(col_vals)
+            errors += 9 - len(un)
+        for tab_vals in tab_values:
+            un = set(tab_vals)
+            errors += 9 - len(un)
+                
+        print("Count of sudoku entries different from label- %s"%(count))
+        print("Count of sudoku violations- %s"%(errors))
+
+    if count == 0:
+        print("value achieved at step ", i)
+        break
+        
 
 ### make the table
 for datanode in program.populate(trainreader):
-    print(datanode)
+    datanode.inferILPResults(empty_entry_label, fun=None)
+
+    print("Datanode %s"%(datanode))
     table = torch.zeros(9, 9)
-    
+    ilpTable = torch.zeros(9, 9)
+
     entries = datanode.getChildDataNodes(conceptName=empty_entry)
     for entry in entries:
         t = entry.getAttribute(empty_entry_label, 'local/argmax')
         predicted = (t == 1).nonzero(as_tuple=True)[0].item() + 1
         table[entry.getAttribute('rows').item()][entry.getAttribute('cols').item()] = predicted
-        print(predicted)
-        if entry.getAttribute('fixed').item() == 1:
-            assert entry.getAttribute('val').item() == predicted
-        print("---")
+        
+        ilpPredicted = entry.getAttribute(empty_entry_label, 'ILP').argmax(dim=-1).item() + 1
+        ilpTable[entry.getAttribute('rows').item()][entry.getAttribute('cols').item()] = ilpPredicted
+
+    print("Argmax Predicted Table")
+    testSudokuPrediction(entries, predictionP = table)
+    
+    print("\n ILP Predicted Table")
+    testSudokuPrediction(entries, predictionP = ilpTable)
     break
     
-
-print(table)
\ No newline at end of file
diff --git a/regr/graph/dataNode.py b/regr/graph/dataNode.py
index 207072da8..f745dcf68 100644
--- a/regr/graph/dataNode.py
+++ b/regr/graph/dataNode.py
@@ -65,7 +65,9 @@
 _DataNodeBulder__Logger.addHandler(ch)
 # Don't propagate
 _DataNodeBulder__Logger.propagate = False
-        
+
+_DataNodeBulder__Logger.info('--- Starting new run ---')
+
 # Class representing single data instance with relation links to other data nodes
 class DataNode:
     def __init__(self, instanceID = None, instanceValue = None, ontologyNode = None, relationLinks = {}, attributes = {}):
@@ -84,6 +86,8 @@ def __init__(self, instanceID = None, instanceValue = None, ontologyNode = None,
             self.attributes = attributes                 # Dictionary with node's attributes
         else:
             self.attributes = {}
+            
+        self.current_device = 'auto'
                      
     class DataNodeError(Exception):
         pass
@@ -787,6 +791,8 @@ def getRelationAttrNames(self, conceptRelation, usedGraph = None):
     #     path can contain eqL statement selecting DataNodes from the DataNodes collecting on the path
     def getEdgeDataNode(self, path):
         # Path is empty
+        if isinstance(path, eqL):
+            path = [path]
         if len(path) == 0:
             return [self]
 
@@ -806,24 +812,28 @@ def getEdgeDataNode(self, path):
         else:
             path0 = path[0]
 
+        relDns = None         
         if self.isRelation(path0):
             relDns = self.getDnsForRelation(path0)
-        else:
+        else: # if not relation then has to be attribute in eql
             attributeValue = self.getAttribute(path[0].e[1]).item()
-            if attributeValue == 1:
-                attributeValue = True
-            else:
-                attributeValue = False
-            if attributeValue in  path[0].e[2]:
+            requiredValue = path[0].e[2]
+             
+            if attributeValue in requiredValue:
+                return [self]
+            elif (True in  requiredValue ) and attributeValue == 1:
                 return [self]
+            elif (False in  requiredValue ) and attributeValue == 0:
+                attributeValue = False
             else:
                 return [None]
-                    
+          
+        # Check if it is a valid relation link  with not empty set of connected datanodes      
         if relDns is None or len(relDns) == 0 or relDns[0] is None:
             return [None]
             relDns = []
             
-        # Filter DataNode through eqL
+        # if eqL then filter DataNode  
         if isinstance(path[0], eqL):
             _cDns = []
             for cDn in relDns:
@@ -1062,23 +1072,7 @@ def inferLocal(self):
                     continue
                 
                 v = dn.getAttribute(c[0])
-                
-                if v is None:
-                    continue
-                elif not torch.is_tensor(v):
-                    continue
-                
-                vClone = torch.clone(v).double()
-                tExp = torch.exp(vClone)
-                for i, e in enumerate(tExp):
-                    if e == float("inf"):
-                        tExp[i] = 1.0
-                        
-                tExpSum = torch.sum(tExp).item()
-            
-                vSoftmax = [(tExp[i]/tExpSum).item() for i in range(len(v))]
-                    
-                vSoftmaxT = torch.as_tensor(vSoftmax) 
+                vSoftmaxT = torch.nn.functional.softmax(v, dim=-1)
                 
                 # Replace nan with 1/len
                 #for i, s in enumerate(vSoftmaxT):
@@ -1143,12 +1137,12 @@ def verifySelection(self, *_conceptsRelations):
     # T-norms: L - Lukasiewicz, G - Godel, P - Product
     #tnorms = ['L', 'G', 'P']
     tnormsDefault = 'P'
-    def calculateLcLoss(self, tnorm=tnormsDefault, sample = False, sampleSize = 0):
+    def calculateLcLoss(self, tnorm=tnormsDefault, sample = False, sampleSize = 0, sampleGlobalLoss = False):
         
         myilpOntSolver, _ = self.__getILPSolver(conceptsRelations = self.collectConceptsAndRelations())
 
         self.inferLocal()
-        lcResult = myilpOntSolver.calculateLcLoss(self, tnorm = tnorm, sample = sample, sampleSize = sampleSize)
+        lcResult = myilpOntSolver.calculateLcLoss(self, tnorm = tnorm, sample = sample, sampleSize = sampleSize, sampleGlobalLoss = sampleGlobalLoss)
         
         return lcResult
 
@@ -2246,7 +2240,7 @@ def addBatchRootDN(self):
             raise ValueError('DataNode Builder has no DataNode started yet')   
         
     # Method returning constructed DataNode - the fist in the list
-    def getDataNode(self):
+    def getDataNode(self, device='auto'):
         self.__addGetDataNodeCounter()
         
         if dict.__contains__(self, 'dataNode'):
@@ -2255,6 +2249,13 @@ def getDataNode(self):
             if len(_dataNode) > 0:  
                 returnDn = _dataNode[0]
                 
+                # Set the torch device
+                returnDn.current_device = device
+                if returnDn.current_device=='auto': # if not set use cpu or cuda if available
+                    returnDn.current_device = 'cpu'
+                    if torch.cuda.is_available():
+                        returnDn.current_device = 'cuda'
+                    
                 if len(_dataNode) == 1:
                     _DataNodeBulder__Logger.info('Returning dataNode with id %s of type %s'%(returnDn.instanceID,returnDn.getOntologyNode().name))
                 else:
diff --git a/regr/graph/logicalConstrain.py b/regr/graph/logicalConstrain.py
index 4f8d5fdbc..d36e63e51 100644
--- a/regr/graph/logicalConstrain.py
+++ b/regr/graph/logicalConstrain.py
@@ -3,15 +3,17 @@
 from regr.graph import Concept
 
 import logging
+import torch
 myLogger = logging.getLogger(ilpConfig['log_name'])
 ifLog =  ilpConfig['ifLog']
         
 V = namedtuple("V", ['name', 'v'], defaults= [None, None])
 
 class LogicalConstrain:
-    def __init__(self, *e, p=100, active = True, name = None):
+    def __init__(self, *e, p=100, active = True, sampleEntries  = False, name = None):
         self.headLC = True # Indicate that it is head constrain and should be process individually
         self.active = active
+        self.sampleEntries = sampleEntries
         
         if not e:
             myLogger.error("Logical Constrain initialized is empty")
@@ -140,7 +142,7 @@ def strEs(self):
     
     #------------
     
-    def createSingleVarILPConstrains(self, lcName, lcFun, model, v, headConstrain = False):  
+    def createSingleVarILPConstrains(self, lcName, lcFun, model, v, headConstrain):  
         singleV = []
                     
         if len(v) != 1:
@@ -238,7 +240,7 @@ def _collectILPVariableSetups(self, lcVariableName, lcVariableNames, v, lcVars =
             return newLcVars
 
     # Method building ILP constraints
-    def createILPConstrains(self, lcName, lcFun, model, v, headConstrain = False):
+    def createILPConstrains(self, lcName, lcFun, model, v, headConstrain):
         if len(v) < 2:
             myLogger.error("%s Logical Constrain created with %i sets of variables which is less then two"%(lcName, len(v)))
             return None
@@ -275,15 +277,18 @@ def createILPConstrains(self, lcName, lcFun, model, v, headConstrain = False):
                 
             rVars.append(tVars)
         
-        # Return results from created ILP constraints - None if headConstrain is True or no ILP constraint created, ILP variable representing the value of ILP constraint, loss calculated
+        # Return results from created ILP constraints - 
+        # None if headConstrain is True or no ILP constraint created, ILP variable representing the value of ILP constraint, loss calculated
         return rVars
     
-    def createILPCount(self, model, myIlpBooleanProcessor, v, headConstrain = False, cOperation = None, cLimit = 1, logicMethodName = "COUNT"):         
+    def createILPCount(self, model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, integrate, logicMethodName = "COUNT"):         
         try:
             lcVariableNames = [e for e in iter(v)]
         except StopIteration:
             pass
         
+        if cLimit== None:
+            cLimit
         lcVariableName0 = lcVariableNames[0] # First variable
         lcVariableSet0 =  v[lcVariableName0]
 
@@ -297,109 +302,125 @@ def createILPCount(self, model, myIlpBooleanProcessor, v, headConstrain = False,
                 var.extend(v[currentV][i])
                 
             if len(var) == 0:
-                if not headConstrain:
+                if not (headConstrain or integrate):
                     varsSetup.append([None])
                     
                 continue
             
-            if headConstrain:
+            if headConstrain or integrate:
                 varsSetup.extend(var)
             else:
                 varsSetup.append(var)
              
         # -- Use ILP variable setup to create constrains   
-        if headConstrain:
-            zVars.append([myIlpBooleanProcessor.countVar(model, *varsSetup, onlyConstrains = headConstrain, limitOp = cOperation, limit=cLimit, logicMethodName = logicMethodName)])
+        if headConstrain or integrate:
+            zVars.append([myIlpBooleanProcessor.countVar(model, *varsSetup, onlyConstrains = headConstrain, limitOp = cOperation, limit=cLimit, 
+                                                         logicMethodName = logicMethodName)])
         else:
             for current_var in varsSetup:
-                zVars.append([myIlpBooleanProcessor.countVar(model, *current_var, onlyConstrains = headConstrain, limitOp = cOperation, limit=cLimit, logicMethodName = logicMethodName)])
+                zVars.append([myIlpBooleanProcessor.countVar(model, *current_var, onlyConstrains = headConstrain, limitOp = cOperation, limit=cLimit, 
+                                                             logicMethodName = logicMethodName)])
            
         if model is not None:
             model.update()
             
         return zVars
         
+
+def use_grad(grad):
+    if not grad:
+        torch.no_grad()
+        
 # ----------------- Logical Single Variable
 
 class notL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createSingleVarILPConstrains("Not", myIlpBooleanProcessor.notVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createSingleVarILPConstrains("Not", myIlpBooleanProcessor.notVar, model, v, headConstrain)
 
 # ----------------- Logical
 
 class andL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createILPConstrains('And', myIlpBooleanProcessor.andVar, model, v, headConstrain)        
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPConstrains('And', myIlpBooleanProcessor.andVar, model, v, headConstrain)        
 
 class orL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False):
-        return self.createILPConstrains('Or', myIlpBooleanProcessor.orVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False):
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPConstrains('Or', myIlpBooleanProcessor.orVar, model, v, headConstrain)
     
 class nandL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createILPConstrains('Nand', myIlpBooleanProcessor.nandVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPConstrains('Nand', myIlpBooleanProcessor.nandVar, model, v, headConstrain)
         
 class ifL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
     
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createILPConstrains('If', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad): #use_grad(myIlpBooleanProcessor.grad):
+            return self.createILPConstrains('If', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
     
 class norL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
     
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createILPConstrains('Nor', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPConstrains('Nor', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
 
 class xorL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
     
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createILPConstrains('Xor', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPConstrains('Xor', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
     
 class epqL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
     
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createILPConstrains('Epq', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False):
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad): 
+            return self.createILPConstrains('Epq', myIlpBooleanProcessor.ifVar, model, v, headConstrain)
      
 # ----------------- Auxiliary
      
 class eqL(LogicalConstrain):
-    def __init__(self, *e, active = True, name = None):
+    def __init__(self, *e, active = True, sampleEntries = False, name = None):
         LogicalConstrain.__init__(self, *e, p=100)
         self.headLC = False
     
 class fixedL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
-        return self.createSingleVarILPConstrains("Fixed", myIlpBooleanProcessor.fixedVar, model, v, headConstrain)
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False):
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad): 
+            return self.createSingleVarILPConstrains("Fixed", myIlpBooleanProcessor.fixedVar, model, v, headConstrain)
     
 # ----------------- Counting
 
 class exactL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
         if isinstance(self.e[-1], int):
             cLimit = self.e[-1]
         else:
@@ -407,24 +428,26 @@ def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False):
 
         cOperation = '=='
         
-        return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, logicMethodName = str(self))
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, integrate, logicMethodName = str(self))
 
 class existsL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
         cLimit = 1
 
         cOperation = '>='
         
-        return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, logicMethodName = str(self))
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, integrate, logicMethodName = str(self))
 
 class atLeastL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
         if isinstance(self.e[-1], int):
             cLimit = self.e[-1]
         else:
@@ -432,13 +455,14 @@ def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False):
             
         cOperation = '>='
         
-        return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, logicMethodName = str(self))
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, integrate, logicMethodName = str(self))
     
 class atMostL(LogicalConstrain):
-    def __init__(self, *e, p=100, active = True, name = None):
-        LogicalConstrain.__init__(self, *e, p=p, active=active, name=name)
+    def __init__(self, *e, p=100, active = True, sampleEntries = False, name = None):
+        LogicalConstrain.__init__(self, *e, p=p, active=active, sampleEntries  = sampleEntries, name=name)
         
-    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False): 
+    def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False, integrate = False): 
         if isinstance(self.e[-1], int):
             cLimit = self.e[-1]
         else:
@@ -446,4 +470,5 @@ def __call__(self, model, myIlpBooleanProcessor, v, headConstrain = False):
             
         cOperation = '<='
         
-        return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, logicMethodName = str(self))
\ No newline at end of file
+        with torch.set_grad_enabled(myIlpBooleanProcessor.grad):
+            return self.createILPCount(model, myIlpBooleanProcessor, v, headConstrain, cOperation, cLimit, integrate, logicMethodName = str(self))
\ No newline at end of file
diff --git a/regr/program/lossprogram.py b/regr/program/lossprogram.py
index ccc7d92af..c26c6ff1e 100644
--- a/regr/program/lossprogram.py
+++ b/regr/program/lossprogram.py
@@ -6,6 +6,7 @@
 from .program import LearningBasedProgram, get_len
 from ..utils import consume, entuple, detuple
 from .model.lossModel import PrimalDualModel, SampleLosslModel
+from .model.base import Mode
 
 # Primal-dual need multiple backward through constraint loss.
 # It requires retain_graph=True.
@@ -75,8 +76,8 @@ def train(
         #     COptim = Optim
         # if COptim is not None and list(self.model.parameters()):
         #     self.copt = COptim(self.model.parameters())
-        if list(self.model.parameters()):
-            self.copt = torch.optim.SGD(self.model.parameters(), lr=c_lr, momentum=c_momentum)
+        if list(self.cmodel.parameters()):
+            self.copt = torch.optim.Adam(self.cmodel.parameters(), lr=c_lr)
         else:
             self.copt = None
             
@@ -109,7 +110,7 @@ def call_epoch(self, name, dataset, epoch_fn, **kwargs):
 
                 metricName = 'loss'
                 metricResult = self.model.loss
-
+                    
                 if self.dbUpdate is not None:
                     self.dbUpdate(desc, metricName, metricResult)
                     
@@ -159,6 +160,7 @@ def train_epoch(
         c_session={},
         **kwargs):
         assert c_session
+        self.model.mode(Mode.TRAIN)
         iter = c_session['iter']
         c_update_iter = c_session['c_update_iter']
         c_update_freq = c_session['c_update_freq']
@@ -252,4 +254,81 @@ class SampleLossProgram(LossProgram):
     logger = logging.getLogger(__name__)
 
     def __init__(self, graph, Model, beta=1, **kwargs):
-        super().__init__(graph, Model, CModel=SampleLosslModel, beta=beta, **kwargs)
\ No newline at end of file
+        super().__init__(graph, Model, CModel=SampleLosslModel, beta=beta, **kwargs)
+
+
+    def train(
+        self,
+        training_set,
+        valid_set=None,
+        test_set=None,
+        # COptim=None,  # SGD only
+        c_lr=0.05,
+        c_momentum=0.9,
+        c_warmup_iters=100,  # warmup
+        c_freq=10,
+        c_freq_increase=5,  # d
+        c_freq_increase_freq=1,
+        c_lr_decay=4,  # strategy
+        c_lr_decay_param=1,  # param in the strategy
+        **kwargs):
+        
+        return super().train(
+            training_set=training_set,
+            valid_set=valid_set,
+            test_set=test_set,
+            c_lr=c_lr,
+            c_momentum=c_momentum,
+            c_warmup_iters=c_warmup_iters,  # warmup
+            c_freq=c_freq,
+            c_freq_increase=c_freq_increase,  # d
+            c_freq_increase_freq=c_freq_increase_freq,
+            c_lr_decay=c_lr_decay,  # strategy
+            c_lr_decay_param=c_lr_decay_param,  # param in the strategy
+            **kwargs)
+
+
+    def train_epoch(
+        self, dataset,
+        c_warmup_iters=0,  # warmup
+        c_session={},
+        **kwargs):
+        self.model.mode(Mode.TRAIN)
+        assert c_session
+        iter = c_session['iter']
+        self.model.train()
+        self.model.reset()
+        self.model.train()
+        self.model.reset()
+        for data in dataset:
+            if self.opt is not None:
+                self.opt.zero_grad()
+            if self.copt is not None:
+                self.copt.zero_grad()
+            mloss, metric, *output = self.model(data)  # output = (datanode, builder)
+            if iter < c_warmup_iters:
+                loss = mloss
+            else:
+                closs, *_ = self.cmodel(output[1])
+                if torch.is_tensor(closs):
+                    loss = mloss + self.beta * closs
+                else:
+                    loss = mloss
+            if self.opt is not None and loss:
+                loss.backward()
+                # for name, param in self.model.named_parameters():
+                #     if param.requires_grad:
+                #         print (name, param.grad)
+
+                self.opt.step()
+                iter += 1
+            
+            if (
+                self.copt is not None and
+                loss
+            ):
+                self.copt.step()
+            
+            yield (loss, metric, *output[:1])
+
+        c_session['iter'] = iter
\ No newline at end of file
diff --git a/regr/program/model/ilpu.py b/regr/program/model/ilpu.py
index 649f8edc5..bc7728731 100644
--- a/regr/program/model/ilpu.py
+++ b/regr/program/model/ilpu.py
@@ -17,7 +17,7 @@ def poi_loss(self, data_item, prop, sensors):
         builder = data_item
         if (builder.needsBatchRootDN()):
             builder.addBatchRootDN()
-        datanode = builder.getDataNode()
+        datanode = builder.getDataNode(device=self.device)
         concept = prop.sup
         values = []
         try:
diff --git a/regr/program/model/iml.py b/regr/program/model/iml.py
index 1be9e01af..0e584f5ac 100644
--- a/regr/program/model/iml.py
+++ b/regr/program/model/iml.py
@@ -17,7 +17,7 @@ def poi_loss(self, data_item, prop, sensors):
         builder = data_item
         if (builder.needsBatchRootDN()):
             builder.addBatchRootDN()
-        datanode = builder.getDataNode()
+        datanode = builder.getDataNode(device=self.device)
         concept = prop.sup
         values = []
         try:
diff --git a/regr/program/model/lossModel.py b/regr/program/model/lossModel.py
index d4f0cab69..0c3941304 100644
--- a/regr/program/model/lossModel.py
+++ b/regr/program/model/lossModel.py
@@ -13,12 +13,13 @@ class LossModel(torch.nn.Module):
 
     def __init__(self, graph, 
                  tnorm=DataNode.tnormsDefault, 
-                 sample = False, sampleSize = 0, sampleGlobalLoss = False):
+                 sample = False, sampleSize = 0, sampleGlobalLoss = False, device='auto'):
         super().__init__()
         self.graph = graph
         self.build = True
         
         self.tnorm = tnorm
+        self.device = device
         
         self.sample = sample
         self.sampleSize = sampleSize
@@ -63,7 +64,7 @@ def forward(self, builder, build=None):
         
         if (builder.needsBatchRootDN()):
             builder.addBatchRootDN()
-        datanode = builder.getDataNode()
+        datanode = builder.getDataNode(device=self.device)
         
         # Call the loss calculation returns a dictionary, keys are matching the constraints
         constr_loss = datanode.calculateLcLoss(tnorm=self.tnorm, sample=self.sample, sampleSize = self.sampleSize)
@@ -90,11 +91,163 @@ def forward(self, builder, build=None):
 class PrimalDualModel(LossModel):
     logger = logging.getLogger(__name__)
 
-    def __init__(self, graph, tnorm=DataNode.tnormsDefault):
-        super().__init__(graph, tnorm=tnorm)
+    def __init__(self, graph, tnorm=DataNode.tnormsDefault, device='auto'):
+        super().__init__(graph, tnorm=tnorm, device=device)
         
-class SampleLosslModel(LossModel):
+class SampleLosslModel(torch.nn.Module):
     logger = logging.getLogger(__name__)
 
-    def __init__(self, graph, sample = False, sampleSize = 0, sampleGlobalLoss = False):
-        super().__init__(graph, sample=sample, sampleSize=sampleSize, sampleGlobalLoss=sampleGlobalLoss)
+    # def __init__(self, graph, sample = False, sampleSize = 0, sampleGlobalLoss = False):
+    #     super().__init__(graph, sample=sample, sampleSize=sampleSize, sampleGlobalLoss=sampleGlobalLoss)
+
+    def __init__(self, graph, 
+                 tnorm=DataNode.tnormsDefault, 
+                 sample = False, sampleSize = 0, sampleGlobalLoss = False, device='auto'):
+        super().__init__()
+        self.graph = graph
+        self.build = True
+        
+        self.tnorm = tnorm
+        self.device = device
+        
+        self.sample = sample
+        self.sampleSize = sampleSize
+        self.sampleGlobalLoss = sampleGlobalLoss
+        
+        self.constr = OrderedDict(graph.logicalConstrainsRecursive)
+        nconstr = len(self.constr)
+        if nconstr == 0:
+            warnings.warn('No logical constraint detected in the graph. '
+                          'PrimalDualModel will not generate any constraint loss.')
+            
+        self.lmbd = torch.nn.Parameter(torch.zeros(nconstr).float())
+        self.lmbd_index = {}
+
+        self.iter_step = 0
+        self.warmpup = 80
+        
+        for i, (key, lc) in enumerate(self.constr.items()):
+            self.lmbd_index[key] = i
+            
+        self.reset_parameters()
+        self.loss = MacroAverageTracker(lambda x:x)
+
+    def reset_parameters(self):
+        torch.nn.init.constant_(self.lmbd, 0.0)
+
+    def reset(self):
+        if isinstance(self.loss, MetricTracker):
+            self.loss.reset()
+
+    def get_lmbd(self, key):
+        if self.lmbd[self.lmbd_index[key]] < 0:
+            with torch.no_grad():
+                self.lmbd[self.lmbd_index[key]] = 0
+        return self.lmbd[self.lmbd_index[key]]
+
+    def forward(self, builder, build=None):
+        if build is None:
+            build = self.build
+        self.iter_step += 1
+            
+        if not build and not isinstance(builder, DataNodeBuilder):
+            raise ValueError('PrimalDualModel must be invoked with `build` on or with provided DataNode Builder.')
+        
+        if (builder.needsBatchRootDN()):
+            builder.addBatchRootDN()
+        
+        self.loss.reset()
+
+        datanode = builder.getDataNode(device=self.device)
+        
+        # Call the loss calculation returns a dictionary, keys are matching the constraints
+        constr_loss = datanode.calculateLcLoss(tnorm=self.tnorm, sample=self.sample, sampleSize = self.sampleSize, sampleGlobalLoss = self.sampleGlobalLoss)
+        import math
+        lmbd_loss = []
+        replace_mul = False
+        
+        key_losses = dict()
+        for key, loss in constr_loss.items():
+            if key not in self.constr:
+                continue
+            # loss_value = loss['loss']
+            epsilon = 0.0
+            key_loss = 0
+            new_eps = 0.01
+            for i, lossTensor in enumerate(loss['lossTensor']):
+                lcSuccesses = loss['lcSuccesses'][i]
+                if constr_loss["globalSuccessCountet"] > 0:
+                    lcSuccesses = constr_loss["globalSuccesses"]
+                if lossTensor.sum().item() != 0:
+                    tidx = (lcSuccesses == 1).nonzero().squeeze(-1)
+                    true_val = lossTensor[tidx]
+                    
+                    if true_val.sum().item() != 0: 
+                        if not replace_mul:
+                            loss_value = true_val.sum() / lossTensor.sum()
+                            loss_value = epsilon - ( -1 * torch.log(loss_value) )
+                            # loss_value = -1 * torch.log(loss_value) 
+                            if self.iter_step < self.warmpup:
+                                with torch.no_grad():
+                                    min_val = loss_value
+                            else:
+                                min_val = -1
+                            # min_val = -1
+                            # with torch.no_grad():
+                            #     min_val = loss_value
+                            loss_ = min_val * loss_value
+                            key_loss += loss_
+                        else:
+                            loss_value = true_val.logsumexp(dim=0) - lossTensor.logsumexp(dim=0)
+                            key_loss += -1 * loss_value
+
+                    else:
+                        loss_ = 0
+                    
+                    
+
+                    # if loss['lossTensor'].nansum().item() <= 1:
+                    #     loss_value = loss['lossTensor']
+                    #     # loss_value = loss_value[loss_value.nonzero()].squeeze(-1)
+                    # else:
+                    #     _idx = []
+                    #     for i in torch.unique(loss['lossTensor']):
+                    #         _idx.append((loss['lossTensor'] == i.item()).nonzero(as_tuple=True)[0][0].item())
+                    #     loss_value = loss['lossTensor'][_idx]
+                    #     # loss_value = torch.unique(loss['lossTensor'])
+
+                    # loss_value = torch.log(loss_value.sum())
+                    # loss_ = -1 * (loss_value)
+                    # self.loss[key](loss_)
+                    # lmbd_loss.append(loss_) 
+                    
+                else:
+                    loss_ = 0
+
+            epsilon = 1e-2
+            key_loss = max(key_loss - epsilon, 0) 
+            if key_loss != 0:  
+                key_losses[key] = key_loss
+                # self.loss[key](key_loss)
+                # lmbd_loss.append(key_loss) 
+                    
+        all_losses = [key_losses[key] for key in key_losses]
+        if all_losses:
+            all_losses = torch.stack(all_losses)
+
+            satisfied_num = len( set(constr_loss.keys()) - set(key_losses.keys()) )
+            unsatisfied_num = len(set(constr_loss.keys())) - satisfied_num
+            self.logger.info(f'-- number of satisfied constraints are {satisfied_num}')
+            self.logger.info(f'-- number of unstatisfied constraints are {unsatisfied_num}')
+            for key in key_losses:
+                if replace_mul:
+                    loss_val = (key_losses[key] / all_losses.sum()) * key_losses[key]
+                else:
+                    loss_val = key_losses[key]
+                self.loss[key](loss_val)
+                lmbd_loss.append(loss_val) 
+                
+            lmbd_loss = sum(lmbd_loss)
+        
+        # (*out, datanode, builder)
+        return lmbd_loss, datanode, builder
diff --git a/regr/program/model/primaldual.py b/regr/program/model/primaldual.py
index fde327e6b..3090ad4e4 100644
--- a/regr/program/model/primaldual.py
+++ b/regr/program/model/primaldual.py
@@ -12,11 +12,12 @@
 class PrimalDualModel(torch.nn.Module):
     logger = logging.getLogger(__name__)
 
-    def __init__(self, graph, tnorm=DataNode.tnormsDefault):
+    def __init__(self, graph, tnorm=DataNode.tnormsDefault, device='auto'):
         super().__init__()
         self.graph = graph
         self.build = True
         self.tnorm = tnorm
+        self.device = device
         constr = OrderedDict(graph.logicalConstrainsRecursive)
         nconstr = len(constr)
         if nconstr == 0:
@@ -51,7 +52,7 @@ def forward(self, builder, build=None):
             raise ValueError('PrimalDualModel must be invoked with `build` on.')
         if (builder.needsBatchRootDN()):
             builder.addBatchRootDN()
-        datanode = builder.getDataNode()
+        datanode = builder.getDataNode(device=self.device)
         # call the loss calculation
         # returns a dictionary, keys are matching the constraints
         constr_loss = datanode.calculateLcLoss(tnorm=self.tnorm)
diff --git a/regr/program/model/pytorch.py b/regr/program/model/pytorch.py
index ad1e4f38d..df1c19ab5 100644
--- a/regr/program/model/pytorch.py
+++ b/regr/program/model/pytorch.py
@@ -15,11 +15,12 @@
 
 
 class TorchModel(torch.nn.Module):
-    def __init__(self, graph):
+    def __init__(self, graph, device='auto'):
         super().__init__()
         self.graph = graph
         self.mode(Mode.TRAIN)
         self.build = True
+        self.device = device
 
         for learner in self.graph.get_sensors(TorchLearner):
             self.add_module(learner.fullname, learner.model)
@@ -93,7 +94,7 @@ def forward(self, data_item, build=None):
             if (builder.needsBatchRootDN()):
                 builder.addBatchRootDN()
             *out, = self.populate(builder)
-            datanode = builder.getDataNode()
+            datanode = builder.getDataNode(device=self.device)
             return (*out, datanode, builder)
         else:
             *out, = self.populate(data_item)
@@ -108,8 +109,8 @@ def model_helper(Model, *args, **kwargs):
 
 
 class PoiModel(TorchModel):
-    def __init__(self, graph, poi=None, loss=None, metric=None):
-        super().__init__(graph)
+    def __init__(self, graph, poi=None, loss=None, metric=None, device='auto'):
+        super().__init__(graph, device)
         if poi is None:
             self.poi = self.default_poi()
         else:
@@ -198,7 +199,7 @@ def populate(self, builder, run=True):
             for sensors in self.find_sensors(prop):
                 if self.mode() not in {Mode.POPULATE,}:
                     # calculated any loss or metric
-                    if self.loss:
+                    if self.loss is not None:
                         local_loss = self.poi_loss(builder, prop, sensors)
                         if local_loss is not None:
                             loss += local_loss
@@ -210,8 +211,8 @@ def populate(self, builder, run=True):
         return loss, metric
 
 class SolverModel(PoiModel):
-    def __init__(self, graph, poi=None, loss=None, metric=None, inferTypes=['ILP']):
-        super().__init__(graph, poi=poi, loss=loss, metric=metric)
+    def __init__(self, graph, poi=None, loss=None, metric=None, inferTypes=['ILP'], device='auto'):
+        super().__init__(graph, poi=poi, loss=loss, metric=metric, device=device)
         self.inference_with = []
         self.inferTypes = inferTypes
 
@@ -228,7 +229,7 @@ def inference(self, builder):
         # Check if this is batch
         if (builder.needsBatchRootDN()):
             builder.addBatchRootDN()
-        datanode = builder.getDataNode()
+        datanode = builder.getDataNode(device=self.device)
         # trigger inference
 #         fun=lambda val: torch.tensor(val, dtype=float).softmax(dim=-1).detach().cpu().numpy().tolist()
         for infertype in self.inferTypes:
@@ -246,12 +247,10 @@ def populate(self, builder, run=True):
         data_item = self.inference(builder)
         return super().populate(builder, run=False)
     
-    
-
 
 class PoiModelToWorkWithLearnerWithLoss(TorchModel):
-    def __init__(self, graph, poi=None):
-        super().__init__(graph)
+    def __init__(self, graph, poi=None, device='auto'):
+        super().__init__(graph, device=device)
         if poi is not None:
             self.poi = poi
         else:
@@ -310,12 +309,12 @@ def metric(self):
     
     
 class PoiModelDictLoss(PoiModel):
-    def __init__(self, graph, poi=None, loss=None, metric=None, dictloss=None):
+    def __init__(self, graph, poi=None, loss=None, metric=None, dictloss=None, device='auto'):
         self.loss_tracker = MacroAverageTracker()
         if dictloss:
-            super().__init__(graph, poi=poi, loss=self.loss_tracker, metric=metric)
+            super().__init__(graph, poi=poi, loss=self.loss_tracker, metric=metric, device=device)
         else:
-            super().__init__(graph, poi=poi, loss=loss, metric=metric)
+            super().__init__(graph, poi=poi, loss=loss, metric=metric, device=device)
         
         self.metric_tracker = None
         self.losses = dict()
@@ -337,7 +336,7 @@ def reset(self):
             
     def poi_loss(self, data_item, prop, sensors):
         if self.dictloss:
-            if not self.loss:
+            if self.loss is None:
                 return 0
     #         outs = [sensor(data_item) for sensor in sensors]
             target = None
@@ -347,12 +346,14 @@ def poi_loss(self, data_item, prop, sensors):
                     target = sensor
                 else:
                     pred = sensor
-            if target == None or pred == None:
+            if target == None or pred == None or pred(data_item).shape[0] == 0:
                 return None
             if not str(prop.name) in self.dictloss and not "default" in self.dictloss:
                 return None
             elif not str(prop.name) in self.dictloss:
                 self.losses[pred, target] = self.dictloss["default"](data_item, prop, pred(data_item), target(data_item))
+                if torch.isnan(self.losses[pred, target]).any() :
+                    print("here it is")
                 return self.losses[pred, target]
             else:
                 self.losses[pred, target] = self.dictloss[str(prop.name)](data_item, prop, pred(data_item), target(data_item))
@@ -454,8 +455,8 @@ def populate(self, builder, run=True):
     
     
 class SolverModelDictLoss(PoiModelDictLoss):
-    def __init__(self, graph, poi=None, loss=None, metric=None, dictloss=None, inferTypes=['ILP']):
-        super().__init__(graph, poi=poi, loss=loss, metric=metric, dictloss=dictloss)
+    def __init__(self, graph, poi=None, loss=None, metric=None, dictloss=None, inferTypes=['ILP'], device='auto'):
+        super().__init__(graph, poi=poi, loss=loss, metric=metric, dictloss=dictloss, device=device)
         self.inference_with = []
         self.inferTypes = inferTypes
 
@@ -472,7 +473,7 @@ def inference(self, builder):
         # Check if this is batch
         if (builder.needsBatchRootDN()):
             builder.addBatchRootDN()
-        datanode = builder.getDataNode()
+        datanode = builder.getDataNode(device=self.device)
         # trigger inference
 #         fun=lambda val: torch.tensor(val, dtype=float).softmax(dim=-1).detach().cpu().numpy().tolist()
         for infertype in self.inferTypes:
diff --git a/regr/program/model/torch.py b/regr/program/model/torch.py
index afb105f24..337d2d715 100644
--- a/regr/program/model/torch.py
+++ b/regr/program/model/torch.py
@@ -17,12 +17,13 @@ def all_properties(node):
 
 
 class BaseModel(torch.nn.Module):
-    def __init__(self, graph, loss=None, metric=None):
+    def __init__(self, graph, loss=None, metric=None, device='auto'):
         super().__init__()
         self.graph = graph
         self.loss = loss
         self.metric = metric
         self.mode_ = Mode.TRAIN
+        self.device = device
 
     def mode(self, mode):
         self.mode_ = mode
@@ -59,8 +60,8 @@ class TorchModel(BaseModel):
     BaseSensor = TorchSensor
     BaseLearner = ModuleLearner
 
-    def __init__(self, graph, loss=None, metric=None):
-        super().__init__(graph, loss=loss, metric=metric)
+    def __init__(self, graph, loss=None, metric=None, device='auto'):
+        super().__init__(graph, loss=loss, metric=metric, device=device)
         self.poi = {prop: (output_sensor, target_sensor) for prop, output_sensor, target_sensor in self.find_poi()}
         self.graph.poi = self.poi
         for node in self.graph.traversal_apply(all_properties):
diff --git a/regr/program/program.py b/regr/program/program.py
index 834474f6e..cbed157e7 100644
--- a/regr/program/program.py
+++ b/regr/program/program.py
@@ -189,6 +189,8 @@ def to(self, device='auto'):
             for sensor in self.graph.get_sensors(TorchSensor):
                 sensor.device = self.device
 
+            self.model.device = self.device
+
     def calculateMetricDelta(self, metric1, metric2):
         metricDelta = {}
         for k, v in metric1.value().items():
diff --git a/regr/solver/gurobiILPBooleanMethods.py b/regr/solver/gurobiILPBooleanMethods.py
index 2066a160c..4536edaa2 100644
--- a/regr/solver/gurobiILPBooleanMethods.py
+++ b/regr/solver/gurobiILPBooleanMethods.py
@@ -11,6 +11,8 @@ class gurobiILPBooleanProcessor(ilpBooleanProcessor):
     def __init__(self, _ildConfig = ilpConfig) -> None:
         super().__init__()
                 
+        self.grad = False
+        
         self.myLogger = logging.getLogger(ilpConfig['log_name'])
         self.ifLog =  ilpConfig['ifLog']
 
diff --git a/regr/solver/gurobiILPOntSolver.py b/regr/solver/gurobiILPOntSolver.py
index 21db66207..fa4808115 100644
--- a/regr/solver/gurobiILPOntSolver.py
+++ b/regr/solver/gurobiILPOntSolver.py
@@ -1,4 +1,4 @@
-from time import process_time
+from time import process_time, process_time_ns
 from collections import OrderedDict
 import logging
 # ontology
@@ -8,7 +8,7 @@
 import torch
 
 # Gurobi
-from gurobipy import GRB, Model, Var
+from gurobipy import GRB, Model, Var, Env
 
 from regr.graph.concept import Concept, EnumConcept
 from regr.solver.ilpOntSolver import ilpOntSolver
@@ -17,6 +17,7 @@
 from regr.solver.lcLossSampleBooleanMethods import lcLossSampleBooleanMethods
 
 from regr.graph import LogicalConstrain, V, fixedL
+from _functools import reduce
 class gurobiILPOntSolver(ilpOntSolver):
     ilpSolver = 'Gurobi'
 
@@ -562,12 +563,14 @@ def addOntologyConstrains(self, m, rootDn, *_conceptsRelations):
         
         m.update()
         
-    def addLogicalConstrains(self, m, dn, lcs, p):
+    def addLogicalConstrains(self, m, dn, lcs, p, key = None):
         self.myLogger.info('Starting method')
         
-        key = "/ILP/xP" # to get ILP variable from datanodes
+        if key == None:
+            key = "/ILP/xP" # to get ILP variable from datanodes
         
         for lc in lcs:   
+            startLC = process_time_ns() # timer()
             
             if lc.active:
                 self.myLogger.info('Processing Logical Constrain %s(%s) - %s'%(lc.lcName, lc, lc.strEs()))
@@ -577,26 +580,66 @@ def addLogicalConstrains(self, m, dn, lcs, p):
 
             result = self.__constructLogicalConstrains(lc, self.myIlpBooleanProcessor, m, dn, p, key = key,  lcVariablesDns = {}, headLC = True)
             
+            endLC = process_time_ns() # timer()
+            elapsedInNsLC = endLC - startLC
+            elapsedInMsLC = elapsedInNsLC/1000000
+            
             if result != None and isinstance(result, list):
                 self.myLogger.info('Successfully added Logical Constrain %s'%(lc.lcName))
+                self.myLoggerTime.info('Processing time for Lc %s is: %ims'%(lc.lcName, elapsedInMsLC))
             else:
                 self.myLogger.error('Failed to add Logical Constrain %s'%(lc.lcName))
+                self.myLoggerTime.error('Failed to add Logical Constrain %s'%(lc.lcName))
+
+    def isVariableFixed(self, dn, conceptName, e):
+        
+        for graph in self.myGraph: # Loop through graphs
+            for _, lc in graph.logicalConstrains.items(): # loop trough lcs in the graph
+                if not lc.headLC or not lc.active: # Process only active and head lcs
+                    continue
+                    
+                if type(lc) is not fixedL: # Skip not fixedL lc
+                    continue
+                
+                lcN = lc
+                # TO be finished
+                
+        # For spec
+        if "fixed" in dn.getAttributes() and dn.getAttributes()["fixed"].item() == 1:
+            vDnLabel = self.__getLabel(dn, conceptName).item()
 
+            if vDnLabel == e[1]:
+                return 1
+            else:
+                return 0
+            
+        return None
+        
     def getMLResult(self, dn, conceptName, xPkey, e, p, loss = False, sample = False):
         if dn == None:
             raise Exception("No datanode provided")
-        
-        if sample and 'sample' not in dn.getAttributes():
-            sampleKey = '<' + conceptName + ">/sample" 
+                        
+        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        sampleKey = '<' + conceptName + ">/sample" 
+        if sample and sampleKey not in dn.getAttributes():
             dn.getAttributes()[sampleKey] = {}
-            
+        
         if dn.ontologyNode.name == conceptName:
             if not sample:
                 return 1
             else:
                 sampleSize = p
-                dn.getAttributes()[sampleKey][sampleSize] = torch.ones(sampleSize, dtype=torch.bool)
-                return (dn.getAttributes()[sampleKey][sampleSize], (1.0, dn.getAttributes()[sampleKey][sampleSize]))
+                
+                if sampleSize not in dn.getAttributes()[sampleKey]: 
+                    dn.getAttributes()[sampleKey][sampleSize] = {}
+                    
+                xVarName = "%s_%s_is_%s"%(dn.getOntologyNode(), dn.getInstanceID(), e[1])
+
+                dn.getAttributes()[sampleKey][sampleSize][e[1]] = torch.ones(sampleSize, dtype=torch.bool, device = device)
+                xP = torch.ones(sampleSize, device = device)
+                
+                return (dn.getAttributes()[sampleKey][sampleSize][e[1]], (xP, dn.getAttributes()[sampleKey][sampleSize][e[1]], xVarName))
         
         if xPkey not in dn.attributes:
             if not sample:
@@ -604,34 +647,56 @@ def getMLResult(self, dn, conceptName, xPkey, e, p, loss = False, sample = False
             else:   
                 return ([None], (None, [None]))
         
-        if loss: # Loss calculation
-            try:
-                vDn = dn.getAttribute(xPkey)[e[1]] # Get value for the concept 
-            except IndexError: 
-                vDn = None
-        else: # If ILP inference
-            vDn = dn.getAttribute(xPkey)[p][e[2]] # Get ILP variable for the concept 
+        if not loss: # ------- If ILP inference
+            if "xP" in xPkey:
+                vDn = dn.getAttribute(xPkey)[p][e[2]] # Get ILP variable for the concept
+            else:
+                vDn = dn.getAttribute(xPkey)[e[2]] # Get ILP variable for the concept
+                
+            return vDn # ILP variable for ILP inference
+        
+        # ----- Loss calculation
+        
+        try:
+            vDn = dn.getAttribute(xPkey)[e[1]] # Get value for the concept 
+        except IndexError: 
+            vDn = None
     
-        if torch.is_tensor(vDn) and (len(vDn.shape) == 0 or len(vDn.shape) == 1 and vDn.shape[0] == 1):
-            vDn = vDn.item()  
-             
         if not sample:
             return vDn # Return here if not sample
         
         # --- Generate sample 
+        if torch.is_tensor(vDn) and (len(vDn.shape) == 0 or len(vDn.shape) == 1 and vDn.shape[0] == 1):
+            vDn = vDn.item()  
+             
         sampleSize = p
 
-        if sampleSize not in dn.getAttributes()[sampleKey]: # check if not already generated
+        xVarName = "%s_%s_is_%s"%(dn.getOntologyNode(), dn.getInstanceID(), e[1])
+        
+        isFiexd = self.isVariableFixed(dn, conceptName, e)
+        
+        if isFiexd != None:
+            if isFiexd == 1:
+                xP = torch.ones(sampleSize, device = device)
+            else:
+                xP = torch.zeros(sampleSize, device = device)
+        else:
+            xV = dn.getAttribute(xPkey)
+            xEp = dn.getAttribute(xPkey).expand(sampleSize, len(xV))
+            xP = xEp[:,e[1]]
+                
+        if sampleSize not in dn.getAttributes()[sampleKey]: 
+            dn.getAttributes()[sampleKey][sampleSize] = {}
+            
+        if e[1] not in dn.getAttributes()[sampleKey][sampleSize]:
+            # check if not already generated
             if vDn == None or vDn != vDn:
-                dn.getAttributes()[sampleKey][sampleSize] = [None]
+                dn.getAttributes()[sampleKey][sampleSize][e[1]] = [None]
             else:
                 # Create sample for this concept and sample size
-                t = torch.full((sampleSize,), vDn) # init tensor with probabilities for sampling
-                dn.getAttributes()[sampleKey][sampleSize] = torch.zeros(sampleSize, dtype=torch.bool)
-                
-                torch.bernoulli(t, out = dn.getAttributes()[sampleKey][sampleSize])
-               
-        return (dn.getAttributes()[sampleKey][sampleSize], (vDn, dn.getAttributes()[sampleKey][sampleSize])) # Return sample data and probabilty
+                dn.getAttributes()[sampleKey][sampleSize][e[1]] = torch.bernoulli(xP)
+        
+        return (dn.getAttributes()[sampleKey][sampleSize][e[1]], (xP, dn.getAttributes()[sampleKey][sampleSize][e[1]], xVarName)) # Return sample data and probability info
                       
     def fixedLSupport(self, _dn, conceptName, vDn, i, m):
         vDnLabel = self.__getLabel(_dn, conceptName).item()
@@ -663,7 +728,7 @@ def fixedLSupport(self, _dn, conceptName, vDn, i, m):
             else:
                 return 0
         
-    def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None, lcVariablesDns = None, headLC = False, loss = False, sample = False):
+    def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None, lcVariablesDns = None, headLC = False, loss = False, sample = False, vNo = None):
         if key == None:
             key = ""
         if lcVariablesDns == None:
@@ -672,8 +737,11 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
         lcVariables = {}
         if sample:
             sampleInfo = {}
-        vNo = 0
+        if vNo == None:
+            vNo = [0]
+        
         firstV = True
+        integrate = False
         
         for eIndex, e in enumerate(lc.e): 
             if  isinstance(e, V):
@@ -685,24 +753,25 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
                     variable = lc.e[eIndex+1]
                 else:
                     if isinstance(e, LogicalConstrain):
-                        variable = V(name="_lc" + str(vNo))
-                        vNo += 1
+                        variable = V(name="_lc" + str(vNo[0]))
+                        vNo[0] += 1
                     else:
                         if firstV:
                             variable = V(name="_x" )
                             firstV = False
                         else:
-                            variable = V(name="_x" + str(vNo), v = ("_x",))
-                            vNo += 1
+                            variable = V(name="_x" + str(vNo[0]), v = ("_x",))
+                            vNo[0] += 1
                     
                 if variable.name:
                     variableName = variable.name
                 else:
-                    variableName = "V" + str(eIndex)
+                    variableName = "V" + str(vNo[0])
+                    vNo[0] += 1
                     
                 if variableName in lcVariables:
-                    newvVariableName = "_x" + str(vNo)
-                    vNo += 1
+                    newvVariableName = "_x" + str(vNo[0])
+                    vNo[0] += 1
                     
                     lcVariablesDns[newvVariableName] = lcVariablesDns[variableName]
                     if None in lcVariablesDns:
@@ -726,37 +795,47 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
                         _dns = dn.findDatanodes(select = rootConcept)
                         dnsList = [[dn] for dn in _dns]
                     else: # Path specified
-                        if len(variable.v) == 0:
-                            self.myLogger.error('The element %s of logical constraint %s has empty part v of the variable'%(conceptName, lc.lcName))
-                            return None
+                        from regr.graph.logicalConstrain import eqL
+                        if not isinstance(variable.v, eqL):
+                            if len(variable.v) == 0:
+                                self.myLogger.error('The element %s of logical constraint %s has empty part v of the variable'%(conceptName, lc.lcName))
+                                return None
                           
                         # -- Prepare paths
                         path = variable.v
                         paths = []
                         
-                        if isinstance(path[0], str) and len(path) == 1:
+                        if isinstance(path, eqL):
+                            paths.append(path)
+                        elif isinstance(path[0], str) and len(path) == 1:
                             paths.append(path)
                         elif isinstance(path[0], str) and not isinstance(path[1], tuple):
                             paths.append(path)
-                        else:
+                        else: # If many paths
                             for i, vE in enumerate(variable.v):
                                 if i == 0 and isinstance(vE, str):
                                     continue
                                 
                                 paths.append(vE)
                                 
+                        pathsCount = len(paths)
+                        
                         # -- Process  paths
                         dnsListForPaths = []
                         for i, v in enumerate(paths):
                             dnsListForPaths.append([])
                             
                             # Get name of the referred variable 
-                            referredVariableName = v[0] 
+                            if isinstance(path, eqL):
+                                referredVariableName = None
+                            else:
+                                referredVariableName = v[0] 
                         
                             if referredVariableName not in lcVariablesDns: # Not yet defined - it has to be the current lc element dataNodes list
                                 rootConcept = dn.findRootConceptOrRelation(conceptName)
                                 _dns = dn.findDatanodes(select = rootConcept)
                                 referredDns = [[dn] for dn in _dns]
+                                integrate = True
                             else: # already defined in the logical constraint from the v part 
                                 referredDns = lcVariablesDns[referredVariableName] # Get DataNodes for referred variables already defined in the logical constraint
                                 
@@ -769,11 +848,14 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
                                         continue
                                     
                                     # -- Get DataNodes for the edge defined by the path part of the v
-                                    _eDns = _rDn.getEdgeDataNode(v[1:]) 
+                                    if isinstance(path, eqL):
+                                        _eDns = _rDn.getEdgeDataNode(v) 
+                                    else:
+                                        _eDns = _rDn.getEdgeDataNode(v[1:]) 
                                     
                                     if _eDns and _eDns[0]:
                                         eDns.extend(_eDns)
-                                    else:
+                                    elif not isinstance(path, eqL):
                                         vNames = [v if isinstance(v, str) else v.name for v in v[1:]]
                                         if lc.__str__() != "fixedL":
                                             self.myLogger.info('The graph node %s has no path %s requested by logical constraint %s for concept %s '%
@@ -782,26 +864,38 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
                                         
                                 dnsListForPaths[i].append(eDns)
                            
-                        # ----------- Fix this - TODO: use all the list -----
-                        dnsList = dnsListForPaths[0]
+                        # -- Select a single dns list or Combine the collected lists of dataNodes based on paths 
+                        dnsList = []
+                        newIntersection = True
+                        if newIntersection:
+                            if pathsCount == 1: # Single path
+                                dnsList = dnsListForPaths[0]
+                            else:
+                                # --- Assume Intersection - TODO: in future use lo if defined to determine if different operation                                
+                                for i in range(len(dnsListForPaths[0])):
+                                    se = [set(dnsListForPaths[item][i]) for item in range(pathsCount)]
+                                    dnsListR = reduce(set.intersection, se)
+                                    dnsList.append(list(dnsListR))
+                        else:
+                            dnsList = dnsListForPaths[0]
                            
-                        # -- Combine the collected lists of dataNodes based on paths 
-                        for l in dnsListForPaths[1:]:
-                            # --- Assume Intersection - TODO: in future use lo if defined to determine if different  operation
-                            _d = []
-                            for i in range(len(l)):
-                                di = []
-                                for x in dnsList[i]:
-                                    if x in l[i]:
-                                        di.append(x)
+                            # -- Combine the collected lists of dataNodes based on paths 
+                            for l in dnsListForPaths[1:]:
+                                # --- Assume Intersection - TODO: in future use lo if defined to determine if different  operation
+                                _d = []
+                                for i in range(len(l)):
+                                    di = []
+                                    for x in dnsList[i]:
+                                        if x in l[i]:
+                                            di.append(x)
+                                            
+                                    if not di:
+                                        di = [None]
                                         
-                                if not di:
-                                    di = [None]
+                                    _d.append(di)
                                     
-                                _d.append(di)
+                                dnsList = _d
                                 
-                            dnsList = _d
-                            
                     # -- Get ILP variables from collected DataNodes for the given element of logical constraint
                     
                     vDns = [] # Stores ILP variables
@@ -839,16 +933,7 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
                                 eT = (e[0], e[2], e[2])
                                 
                                 if sample:
-                                    vDn, vDnSampleInfo = self.getMLResult(_dn, conceptName, xPkey, eT, p, loss = loss, sample=sample)
-                                    
-                                    import time
-                                    start = time.perf_counter()
-                                    vSum = torch.sum(vDn)
-                                    vLen = len(vDn)
-                                    vDnCal = torch.sum(vDn)/len(vDn)
-                                    end = time.perf_counter()
-                                    dur = end - start
-                                    
+                                    vDn, vDnSampleInfo = self.getMLResult(_dn, conceptName, xPkey, eT, p, loss = loss, sample=sample)                                    
                                     _sampleInfoForVariable.append(vDnSampleInfo)
                                 else:
                                     vDn = self.getMLResult(_dn, conceptName, xPkey, eT, p, loss = loss, sample=sample)
@@ -890,10 +975,12 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
                 elif isinstance(e, LogicalConstrain): # -- nested LogicalConstrain - process recursively 
                     self.myLogger.info('Processing Nested Logical Constrain %s(%s) - %s'%(e.lcName, e, e.strEs()))
                     if sample:
-                        vDns, sampleInfoLC = self.__constructLogicalConstrains(e, booleanProcesor, m, dn, p, key = key, lcVariablesDns = lcVariablesDns, headLC = False, loss = loss, sample = sample)
+                        vDns, sampleInfoLC = self.__constructLogicalConstrains(e, booleanProcesor, m, dn, p, key = key, 
+                                                                               lcVariablesDns = lcVariablesDns, headLC = False, loss = loss, sample = sample, vNo=vNo)
                         sampleInfo = {**sampleInfo, **sampleInfoLC} # sampleInfo|sampleInfoLC in python 9
                     else:
-                        vDns = self.__constructLogicalConstrains(e, booleanProcesor, m, dn, p, key = key, lcVariablesDns = lcVariablesDns, headLC = False, loss = loss, sample = sample)
+                        vDns = self.__constructLogicalConstrains(e, booleanProcesor, m, dn, p, key = key, 
+                                                                 lcVariablesDns = lcVariablesDns, headLC = False, loss = loss, sample = sample, vNo=vNo)
                     
                     if vDns == None:
                         self.myLogger.warning('Not found data for %s(%s) nested logical Constrain required to build Logical Constrain %s(%s) - skipping this constraint'%
@@ -916,9 +1003,9 @@ def __constructLogicalConstrains(self, lc, booleanProcesor, m, dn, p, key = None
                 self.myLogger.error('Logical Constrain %s has incorrect element %s'%(lc,e))
                 return None
         if sample:
-            return lc(m, booleanProcesor, lcVariables, headConstrain = headLC), sampleInfo
+            return lc(m, booleanProcesor, lcVariables, headConstrain = headLC, integrate = integrate), sampleInfo
         else:
-            return lc(m, booleanProcesor, lcVariables, headConstrain = headLC)
+            return lc(m, booleanProcesor, lcVariables, headConstrain = headLC, integrate = integrate)
     
     # ---------------
                 
@@ -931,7 +1018,8 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
             return 
         
         start = process_time() # timer()
-        
+
+        gurobiEnv = Env("logs/gurobi.log")
         try:
             if self.reuse_model and self.model:
                 m = self.model['m']
@@ -939,18 +1027,26 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
             else:
                 # Create a new Gurobi model
                 self.myIlpBooleanProcessor.resetCaches()
-                m = Model("decideOnClassificationResult" + str(start))
+                m = Model("decideOnClassificationResult" + str(start), gurobiEnv)
                 m.params.outputflag = 0
                 x = {}
                 
             # Create ILP Variables for concepts and objective
             Q = self.createILPVariables(m, x, dn, *conceptsRelations, dnFun = self.__getProbability, fun=fun, epsilon = epsilon)
                 
+            endVariableInit = process_time() # timer()
+            elapsedVariablesInMs = (endVariableInit - start) *1000
+            self.myLoggerTime.info('ILP Variables Init - time: %ism'%(elapsedVariablesInMs))
+
             if self.model is None:
                 # Add constraints based on ontology and graph definition
                 self.addOntologyConstrains(m, dn, *conceptsRelations)
                 self.addGraphConstrains(m, dn, *conceptsRelations)
                 
+            endGraphAndOntologyConstraints = process_time() # timer()
+            elapsedGandOConstraintsInMs = (endGraphAndOntologyConstraints - endVariableInit) * 1000
+            self.myLoggerTime.info('ILP Graph and Ontology Constraints - time: %ims'%(elapsedGandOConstraintsInMs))
+            
             # ILP Model objective setup
             if Q is None:
                 Q = 0
@@ -978,7 +1074,7 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
                                             
                         if lcP not in _lcP:
                             _lcP[lcP] = []
-                            pUsed = True
+                            pUsed = True # Found p different then default 100
                         
                         _lcP[lcP].append(lc) # Keep constraint with the same p in the list 
             
@@ -987,48 +1083,57 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
             for p in lcP:
                 self.myLogger.info('Found %i logical constraints with p %i - %s'%(len(lcP[p]),p,lcP[p]))
                 self.myLoggerTime.info('Starting ILP inferencing - Found %i logical constraints'%(len(lcP[p])))
-                
+            
             # Search through set of logical constraints for subset satisfying and the mmax/min calculated objective value
             lcRun = {} # Keeps information about subsequent model runs
             ps = [] # List with processed p 
+            
+            #  -----------  Run ILP solver for each set of lcs based on their p
             for p in lcP:
                 ps.append(p)
                 
                 if pUsed:
                     mP = m.copy() # Copy model for this run                    
                     xP = {}
+                    lckey = "/ILP/xP"
+                   
+                    pStart= process_time() # timer()
+                    
+                    for _x in x:
+                        # Map variables to the new copy model
+                        xP[_x] = mP.getVarByName(x[_x].VarName)
+                        
+                        rootConcept = dn.findRootConceptOrRelation(_x[0])
+                        
+                        dns = dn.findDatanodes(select = ((rootConcept,), ("instanceID", _x[2])))  
+                        
+                        if dns:
+                            if _x[1].startswith('Not'):
+                                xPkey = '<' + _x[0].name + '>/ILP/notxP'
+                            else:
+                                xPkey = '<' + _x[0].name + '>/ILP/xP'
+    
+                            if xPkey not in dns[0].attributes:
+                                dns[0].attributes[xPkey] = {}
+                                
+                            if p not in dns[0].attributes[xPkey]:
+                                xkey = '<' + _x[0].name + '>/ILP/x'
+                                if xkey not in dns[0].attributes:
+                                    continue
+                                
+                                xLen = len(dns[0].attributes[xkey])
+                                dns[0].attributes[xPkey][p] = [None] * xLen
+                                
+                            dns[0].attributes[xPkey][p][_x[3]] = mP.getVarByName(x[_x].VarName)
+                            
+                            pEnd= process_time() # timer()
+                            self.myLoggerTime.info('ILP Model init for p %i - time: %fs'%(p, pEnd - pStart))
                 else:
                     mP = m
                     xP = x
                     
-                for _x in x:
-                    # Map variables to the new copy model
-                    xP[_x] = mP.getVarByName(x[_x].VarName)
-                    
-                    rootConcept = dn.findRootConceptOrRelation(_x[0])
-                    
-                    dns = dn.findDatanodes(select = ((rootConcept,), ("instanceID", _x[2])))  
-                    
-                    if dns:
-                        if _x[1].startswith('Not'):
-                            xPkey = '<' + _x[0].name + '>/ILP/notxP'
-                        else:
-                            xPkey = '<' + _x[0].name + '>/ILP/xP'
-
-                        if xPkey not in dns[0].attributes:
-                            dns[0].attributes[xPkey] = {}
-                            
-                        if p not in dns[0].attributes[xPkey]:
-                            xkey = '<' + _x[0].name + '>/ILP/x'
-                            if xkey not in dns[0].attributes:
-                                continue
-                            
-                            xLen = len(dns[0].attributes[xkey])
-                            dns[0].attributes[xPkey][p] = [None] * xLen
-                            
-                        dns[0].attributes[xPkey][p][_x[3]] = mP.getVarByName(x[_x].VarName)
-                   
-                    
+                    lckey = "/ILP/x"
+            
                 # Prepare set with logical constraints for this run
                 lcs = []
                 for _p in lcP:
@@ -1037,32 +1142,39 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
                     if _p == p:
                         break     
     
-                # Add LC constraints to the copy model
+                # ----------- Add LC constraints to the ILP model
+                
+                endLogicalConstraintsPrep = process_time() # timer()
+                elapsedLogicalConstraintsPrepInMs = (endLogicalConstraintsPrep - endGraphAndOntologyConstraints)
+                self.myLoggerTime.info('ILP Logical Constraints Preprocessing - time: %fs'%(elapsedLogicalConstraintsPrepInMs))
                 
                 if pUsed or self.model is None:
-                    self.addLogicalConstrains(mP, dn, lcs, p)
+                    self.addLogicalConstrains(mP, dn, lcs, p, key = lckey) # <--- LC constraints
                     
                     if self.reuse_model:
                         self.model = {}
                         self.model['m'] = mP
                         self.model['x'] = xP
                 
-                    
                 self.myLogger.info('Optimizing model for lCs with probabilities %s with %i ILP variables and %i ILP constraints'%(p,mP.NumVars,mP.NumConstrs))
                 self.myLoggerTime.info('Optimizing model for lCs with probabilities %s with %i ILP variables and %i ILP constraints'%(p,mP.NumVars,mP.NumConstrs))
 
+                endLogicalConstraints = process_time() # timer()
+                elapsedLogicalConstraintsInMs = (endLogicalConstraints - endLogicalConstraintsPrep)
+                self.myLoggerTime.info('ILP Logical Constraints - time: %fs'%(elapsedLogicalConstraintsInMs))
+            
+                #mP.update()
+                #mP.display() 
                 startOptimize = process_time() # timer()
 
-                # Run ILP model - Find solution 
+                # ----------- Run ILP model - Find solution 
                 mP.optimize()
                 mP.update()
-                
-                #mP.display()    
-                
+                                
                 endOptimize = process_time() # timer()
                 elapsedOptimizeInMs = (endOptimize - startOptimize) * 1000
     
-                # Check model run result
+                # ----------- Check model run result
                 solved = False
                 objValue = None
                 if mP.status == GRB.Status.OPTIMAL:
@@ -1095,11 +1207,18 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
                     sys.stdout = log
                     mP.display() 
                     sys.stdout = so
+                    
+                if (not solved):
+                    mP.computeIIS()
+                    mP.write("logs/gurabiInfeasible.ilp")
+
+                #if (self.myLogger.level <= logging.INFO):# and self.myLogger.filter(""):
+                #    mP.write("logs/gurabiInfeasible.lp")
 
                 # Keep result of the model run    
                 lcRun[p] = {'p':p, 'solved':solved, 'objValue':objValue, 'lcs':lcs, 'mP':mP, 'xP':xP, 'elapsedOptimize':elapsedOptimizeInMs}
 
-            # Select model run with the max/min objective value 
+            # ----------- Select model run with the max/min objective value 
             maxP = None
             for p in lcRun:
                 if lcRun[p]['objValue']:
@@ -1111,9 +1230,9 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
                     else:
                         maxP = p
                
-            device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+            device = dn.current_device # torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
-            # If found model - return best result          
+            # ----------- If solution found model - return best result          
             if maxP:
                 self.myLogger.info('Best  solution found for p - %i'%(maxP))
                 #self.myLoggerTime.info('Best  solution found for p - %i'%(maxP))
@@ -1136,12 +1255,22 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
                    
                     for cDn in c_root_dns:
                         dnAtt = cDn.getAttributes()
-                        if xPkey not in dnAtt:
-                            dnAtt[ILPkey] = torch.tensor([float("nan")], device=device) 
-                            self.myLogger.info('Not returning solutions for %s in %sit is nan'%(c[1], cDn))
-                            continue
                         
-                        solution = dnAtt[xPkey][maxP][index].X
+                        if pUsed:
+                            if xPkey not in dnAtt:
+                                dnAtt[ILPkey] = torch.tensor([float("nan")], device=device) 
+                                self.myLogger.info('Not returning solutions for %s in %s it is nan'%(c[1], cDn))
+                                continue
+                        
+                            solution = dnAtt[xPkey][maxP][index].X
+                        else:
+                            if xkey not in dnAtt:
+                                dnAtt[ILPkey] = torch.tensor([float("nan")], device=device) 
+                                self.myLogger.info('Not returning solutions for %s in %s it is nan'%(c[1], cDn))
+                                continue
+                            
+                            solution = dnAtt[xkey][index].X
+                            
                         if solution == 0:
                             solution = 0
                         elif solution == 1: 
@@ -1154,9 +1283,10 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
                             dnAtt[xkey] = torch.empty(c[3], dtype=torch.float)
                        
                         dnAtt[ILPkey][index] = solution
-                        dnAtt[xkey][index] = dnAtt[xPkey][maxP][index]
-                        if xNotPkey in dnAtt:
-                            dnAtt[xNotPkey][index] = dnAtt[xNotPkey][maxP][index]
+                        if pUsed: 
+                            dnAtt[xkey][index] = dnAtt[xPkey][maxP][index]
+                            if xNotPkey in dnAtt:
+                                dnAtt[xNotPkey][index] = dnAtt[xNotPkey][maxP][index]
 
                         ILPV = dnAtt[ILPkey][index]
                         if ILPV == 1:
@@ -1171,26 +1301,121 @@ def calculateILPSelection(self, dn, *conceptsRelations, fun=None, epsilon = 0.00
             
             raise
            
+        endResultPrep = process_time() # timer()
+        elapsedResultPrepInMs = (endResultPrep - endOptimize) *1000
+        self.myLoggerTime.info('ILP Preparing Return Results - time: %ims'%(elapsedResultPrepInMs))
+            
+        self.myLogger.info('')
+
         end = process_time() # timer()
         elapsedInS = end - start
         
-        self.myLogger.info('')
-        
-        self.myLogger.info('End ILP Inferencing - elapsed time: %is'%(elapsedInS))
-        self.myLoggerTime.info('End ILP Inferencing - elapsed time: %is'%(elapsedInS))
+        if elapsedInS > 1:
+            self.myLogger.info('End ILP Inferencing - total time: %is'%(elapsedInS))
+            self.myLoggerTime.info('End ILP Inferencing - total time: %is'%(elapsedInS))
+        else:
+            elapsedInMs = (end - start) *1000
+            self.myLogger.info('End ILP Inferencing - total time: %ims'%(elapsedInMs))
+            self.myLoggerTime.info('End ILP Inferencing - total time: %ims'%(elapsedInMs))
+            
         self.myLogger.info('')
         self.myLoggerTime.info('')
         
-        # Return
+        # ----------- Return
         return
 
+    def eliminateDuplicateSamples(self, lcVariables, sampleSize, currentDevice):
+        variablesSamples = [lcVariables[v][1] for v in lcVariables]
+                    
+        variablesSamplesT = torch.stack(variablesSamples)
+        
+        uniqueSampleIndex = OrderedDict()
+        
+        for i in range(sampleSize):
+            currentS = variablesSamplesT[:,i]
+            
+            currentSHash = hash(currentS.cpu().detach().numpy().tobytes())
+            
+            if currentSHash in uniqueSampleIndex:
+                continue
+                
+                # For testing  - not used now
+                if torch.equal(currentS, variablesSamplesT[:,uniqueSampleIndex[currentSHash]]):
+                    continue
+                else:
+                    raise Exception("HashWrong")
+            else:
+                uniqueSampleIndex[currentSHash] = i
+            
+        va = list(uniqueSampleIndex.values())
+            
+        newSampleSize = len(va)
+    
+        indices = torch.tensor(va, device = currentDevice)
+        #x = torch.arange(6).view(2,3)
+        Vs = torch.index_select(variablesSamplesT, dim=1, index=indices)
+        
+        return newSampleSize, indices, Vs
+
+    def calulateSampleLossForVariable(self, lcVariables, lcSuccesses, sampleSize, currentDevice, eliminateDuplicateSamples, replace_mul=False):
+        lcSampleSize = sampleSize
+        if eliminateDuplicateSamples:
+            lcSampleSize, indices, Vs = self.eliminateDuplicateSamples(lcVariables, sampleSize, currentDevice)
+
+        # Calculate loss value
+        if eliminateDuplicateSamples: 
+            lossTensor = torch.index_select(lcSuccesses, dim=0, index=indices)
+        else:
+            # lossTensor = torch.clone(lcSuccesses)
+            if replace_mul:
+                lossTensor = torch.zeros(lcSuccesses.shape).to(currentDevice)
+            else:
+                lossTensor = torch.ones(lcSuccesses.shape).to(currentDevice)
+
+            #lossTensor = countSuccesses.div_(len(lossList))
+            
+        for i, v in enumerate(lcVariables):
+            currentV = lcVariables[v]
+            
+            if eliminateDuplicateSamples:
+                P = currentV[0][:lcSampleSize] # Tensor with the current variable p (v[0])
+            else:
+                P = currentV[0]
+            oneMinusP = torch.sub(torch.ones(lcSampleSize, device=currentDevice), P) # Tensor with the current variable 1-p
+            
+            if eliminateDuplicateSamples:
+                S = Vs[i, :] #currentV[1] # Sample for the current Variable
+            else:
+                S = currentV[1]
+                
+            if isinstance(S, list):
+                continue
+            
+            notS = torch.sub(torch.ones(lcSampleSize, device=currentDevice), S.float()) # Negation of Sample
+            
+            pS = torch.mul(P, S) # Tensor with p multiply by True variable sample
+            oneMinusPS = torch.mul(oneMinusP, notS) # Tensor with 1-p multiply by False variable sample
+            
+            cLoss = torch.add(pS, oneMinusPS) # Sum of p and 1-p tensors
+                                            
+            # Multiply the loss
+            if replace_mul:
+                lossTensor = lossTensor + torch.log(cLoss)
+            else:
+                lossTensor.mul_(cLoss)
+        
+        # if replace_mul:
+        #     lossTensor = torch.exp(lossTensor)
+        return lossTensor, lcSampleSize
+            
     # -- Calculated loss values for logical constraints
-    def calculateLcLoss(self, dn, tnorm='L', sample = False, sampleSize = 0):
+    def calculateLcLoss(self, dn, tnorm='L', sample = False, sampleSize = 0, sampleGlobalLoss = False):
         start = process_time() # timer()
 
         m = None 
         p = 0
-        
+        currentDevice = dn.current_device
+
         if sample: 
             if sampleSize <= 0: 
                 raise Exception("Sample size is not incorrect - %i"%(sampleSize))
@@ -1203,16 +1428,20 @@ def calculateLcLoss(self, dn, tnorm='L', sample = False, sampleSize = 0):
         else:
             myBooleanMethods = self.myLcLossBooleanMethods
             self.myLcLossBooleanMethods.setTNorm(tnorm)
+            
             self.myLogger.info('Calculating loss ')
             self.myLoggerTime.info('Calculating loss ')
 
-
+        myBooleanMethods.current_device = dn.current_device
+        
         key = "/local/softmax"
         
         lcCounter = 0 # Count processed lcs
         lcLosses = {}
         for graph in self.myGraph: # Loop through graphs
             for _, lc in graph.logicalConstrains.items(): # loop trough lcs in the graph
+                startLC = process_time_ns() # timer()
+
                 if not lc.headLC or not lc.active: # Process only active and head lcs
                     continue
                     
@@ -1222,132 +1451,217 @@ def calculateLcLoss(self, dn, tnorm='L', sample = False, sampleSize = 0):
                 lcCounter +=  1
                 self.myLogger.info('Processing Logical Constrain %s(%s) - %s'%(lc.lcName, lc, lc.strEs()))
                 
+                lcName = lc.lcName
+                    
+                lcLosses[lcName] = {}
+                current_lcLosses = lcLosses[lcName]
+                
                 # Calculate loss for the given lc
                 if sample:
                     # lossList will contain boolean results for lc evaluation for the given sample element
                     # sampleInfo - will contain list of variable exiting in the given lc with their sample and probabilities
                     lossList, sampleInfo = self.__constructLogicalConstrains(lc, myBooleanMethods, m, dn, p, key = key, lcVariablesDns = {}, headLC = True, loss = True, sample = sample)
+                    current_lcLosses['lossList'] = lossList
+                    current_lcLosses['sampleInfo'] = sampleInfo
                 else:
                     # lossList will contain float result for lc loss calculation
                     lossList = self.__constructLogicalConstrains(lc, myBooleanMethods, m, dn, p, key = key, lcVariablesDns = {}, headLC = True, loss = True, sample = sample)
+                    current_lcLosses['lossList'] = lossList
+                    
+                    
+                endLC = process_time_ns() # timer()
+                elapsedInNsLC = endLC - startLC
+                elapsedInMsLC = elapsedInNsLC/1000000
+                current_lcLosses['elapsedInMsLC'] = elapsedInMsLC
+                            
+        if not sample: # Loss value
+            for currentLcName in lcLosses:
+                startLC = process_time_ns() # timer()
+
+                current_lcLosses = lcLosses[currentLcName]
+                lossList = current_lcLosses['lossList']
                 
-                if not lossList:
-                    continue
+                lossTensor = torch.zeros(len(lossList))#, requires_grad=True) # Entry lcs
+                for i, l in enumerate(lossList):
+                    lossTensor[i] = float("nan")
+                    for entry in l:
+                        if entry is not None:
+                            if lossTensor[i] != lossTensor[i]:
+                                lossTensor[i] = entry
+                            else:
+                                lossTensor[i] += entry
+    
+                current_lcLosses['lossTensor'] = lossTensor
+                current_lcLosses['loss'] = torch.nansum(lossTensor).item()
                 
-                lcName = lc.lcName
+                endLC = process_time_ns() # timer()
+                elapsedInNsLC = endLC - startLC
+                elapsedInMsLC = elapsedInNsLC/1000000
+                current_lcLosses['elapsedInMsLC'] += elapsedInMsLC
+
+                self.myLoggerTime.info('Processing time for Lc %s with %i entries is: %ims'%(lcName, len(lossList),  current_lcLosses['elapsedInMsLC']))
+                [h.flush() for h in self.myLoggerTime.handlers]
+            
+        else: # -----------Sample
+            globalSuccesses = torch.ones(sampleSize, device = currentDevice)
+            
+            for currentLcName in lcLosses:
+                startLC = process_time_ns() # timer()
+
+                current_lcLosses = lcLosses[currentLcName]
+                lossList = current_lcLosses['lossList']
+                
+                sampleInfo = current_lcLosses['sampleInfo']
                     
-                lcLosses[lcName] = {}
-                current_lcLosses = lcLosses[lcName]
-                lossTensor = torch.zeros(len(lossList))
-                lossRateTensor = torch.zeros(len(lossList))
+                successesList = [] # Entry lcs successes
+                lcSuccesses = torch.ones(sampleSize, device = currentDevice) # Consolidated successes for all the entry lcs
+                lcVariables = {} # Unique variables used in all the entry lcs
+                countSuccesses = torch.zeros(sampleSize, device = currentDevice)
+                oneT = torch.ones(sampleSize, device = currentDevice)
                 
-                if not sample: # Loss value
-                    for i, l in enumerate(lossList):
-                        if l[0] is not None:
-                            lossTensor[i] = l[0]
-                        else:
-                            lossTensor[i] = float("nan")
-                else: # -----------Sample
-                    # Prepare data
-                    lossListInt = []
-                    sampleInfoFiltered = []
-                    for i, l in enumerate(lossList):
-                        lossListIntForL = []
-                        sampleInfoFilteredForL = []
-                        for crrentL in l:
-                            if crrentL is None:
-                                lossListIntForL.append(None)
-                                continue
-                            
-                            lossListIntForL.append(crrentL.to(dtype=torch.int, copy=True))
-                            
-                            currentSampleInfo = []
-                            for k in sampleInfo.keys():
-                                for c in sampleInfo[k][i]:
-                                    currentSampleInfo.append(c)
-                            
-                            sampleInfoFilteredForL.append(currentSampleInfo)
+                # Prepare data
+                if len(lossList) == 1:
+                    for currentFailures in lossList[0]:
+                        if currentFailures is None:
+                            successesList.append(None)
+                            continue
+                        
+                        currentSuccesses = torch.sub(oneT, currentFailures.float())
+                        successesList.append(currentSuccesses)
                             
-                        lossListInt.append(lossListIntForL)
-                        sampleInfoFiltered.append(sampleInfoFilteredForL)
+                        lcSuccesses.mul_(currentSuccesses)
+                        globalSuccesses.mul_(currentSuccesses)
+                        countSuccesses.add_(currentSuccesses)
                         
-                    # Calculate loss value
-                    for i, l in enumerate(lossListInt):
-                        countFailures = 0
-                        for currentFailures in l:    
-                            if currentFailures == None:
-                                lossTensor[i] = float("nan")
-                                lossRateTensor[i] = float("nan")
+                    # Collect lc variable
+                    for k in sampleInfo.keys():
+                        for c in sampleInfo[k]:
+                            if not c:
+                                continue
+                            c = c[0]
+                            if len(c) > 2:                                    
+                                if c[2] not in lcVariables:
+                                    lcVariables[c[2]] = c
+                else:
+                    for i, l in enumerate(lossList):
+                        for currentFailures in l:
+                            if currentFailures is None:
+                                successesList.append(None)
                                 continue
                             
-                            # Calculate Loss rate in sample
-                            countFailures += torch.sum(currentFailures).item()
-                            
-                            # Calculate loss value for sample loss
-                            currentLossT = None
-                            for _, vList in enumerate(sampleInfoFiltered[i]): # Loop trough variables in the lc
-                                for v in vList:
-                                    P = torch.full([len(l)], v[0]) # Tensor with the current variable p (v[0])
-                                    oneMinusP = torch.sub(torch.ones(len(l)), P) # Tensor with the current variable 1-p
-                                    
-                                    pS = torch.mul(P, v[1]) # Tensor with p multiply by True variable sample (v[1])
-                                    oneMinusPS = torch.mul(oneMinusP, torch.logical_not(v[1])) # Tensor with 1-p multiply by False variable sample (v[1])
-                                    
-                                    pSum = torch.add(pS, oneMinusPS) # Sum of p and 1-p tensors
-                                    
-                                    currentSuccesses = torch.logical_not(currentFailures)
-                                    cLoss = torch.mul(pSum, currentSuccesses) # Chose successes values
-                                    
-                                    # Multiply the loss
-                                    if currentLossT == None:
-                                        currentLossT = cLoss
-                                    else:
-                                        currentLossT = torch.mul(currentLossT, cLoss)
-                                
-                            # Collect sample used for calculation
-                            if currentLossT == None:
-                                currentLoss = float("nan")
-                            else:
-                                currentLoss = torch.sum(currentLossT).item()
+                            currentSuccesses = torch.sub(oneT, currentFailures.float())
+                            successesList.append(currentSuccesses)
                                 
-                            # Collect calculated loss for sample
-                            lossTensor[i] = currentLoss
+                            lcSuccesses.mul_(currentSuccesses)
+                            globalSuccesses.mul_(currentSuccesses)
+                            countSuccesses.add_(currentSuccesses)
+                            
+                        # Collect lc variable
+                        for k in sampleInfo.keys():
+                            for c in sampleInfo[k][i]:
+                                if len(c) > 2:                                        
+                                    if c[2] not in lcVariables:
+                                        lcVariables[c[2]] = c
+                
+                current_lcLosses['successesList'] = successesList
+                current_lcLosses['lcSuccesses'] = lcSuccesses
+                current_lcLosses['lcVariables'] = lcVariables
+                current_lcLosses['countSuccesses'] = countSuccesses
 
-                        lossRateTensor[i] = countFailures
+                endLC = process_time_ns() # timer()
+                elapsedInNsLC = endLC - startLC
+                elapsedInMsLC = elapsedInNsLC/1000000
+                current_lcLosses['elapsedInMsLC'] += elapsedInMsLC
 
-                current_lcLosses['lossTensor'] = lossTensor
-                current_lcLosses['loss'] = torch.nansum(lossTensor).item() / len(lossTensor)
+            lcLosses["globalSuccesses"] = globalSuccesses
+            lcLosses["globalSuccessCountet"] = torch.nansum(globalSuccesses).item()
+            self.myLoggerTime.info('Global success counter is %i '%(lcLosses["globalSuccessCountet"]))
+            
+            
+            for currentLcName in lcLosses:
+                if currentLcName in ["globalSuccessCountet", "globalSuccesses"]:
+                    continue
                 
-                if sample:
-                    current_lcLosses['lossRateTensor'] = lossRateTensor
-                    current_lcLosses['failuresRate'] = torch.nansum(lossRateTensor).item() / len(lossRateTensor)
+                startLC = process_time_ns() # timer()
 
+                current_lcLosses = lcLosses[currentLcName]
+                lossList = current_lcLosses['lossList']
+                
+                successesList = current_lcLosses['successesList']
+                lcSuccesses = current_lcLosses['lcSuccesses']
+                lcVariables = current_lcLosses['lcVariables']
+                        
+                #lcSuccessesSum = torch.sum(lcSuccesses).item()
+                                
+                eliminateDuplicateSamples = False # Eliminate duplicate samples
+                
+                # --- Calculate sample loss for lc variables
+                current_lcLosses['lossTensor'] = []
+                current_lcLosses['lcSuccesses'] = []
+                current_lcLosses['lcVariables'] = []
+                current_lcLosses['loss'] = []
+                
+                # Per each lc entry separately
+                if lc.sampleEntries:
+                    current_lcLosses['lcSuccesses'] = successesList
+                    
+                    for i, l in enumerate(lossList):
+                        currentLcVariables = {}
+                        for k in sampleInfo.keys():
+                            for c in sampleInfo[k][i]:
+                                if len(c) > 2:                                        
+                                    if c[2] not in currentLcVariables:
+                                        currentLcVariables[c[2]] = c
+                                        
+                        
+                        usedLcSuccesses = successesList[i]
+                        if sampleGlobalLoss:
+                            usedLcSuccesses = globalSuccesses
+                        currentLossTensor, _ = self.calulateSampleLossForVariable(currentLcVariables, usedLcSuccesses, sampleSize, currentDevice, eliminateDuplicateSamples)
+                        
+                        current_lcLosses['lossTensor'].append(currentLossTensor)
+                        current_lcLosses['lcVariables'].append(currentLcVariables)
+    
+                        currentLoss = torch.nansum(currentLossTensor).item() # Sum of losses across sample for x|=alfa
+                        current_lcLosses['loss'].append(currentLoss)
+    
+                else: # Regular calculation for all lc entries at once
+                    usedLcSuccesses = lcSuccesses
+                    if sampleGlobalLoss:
+                        usedLcSuccesses = globalSuccesses
+                    lossTensor, lcSampleSize = self.calulateSampleLossForVariable(lcVariables, usedLcSuccesses, sampleSize, currentDevice, eliminateDuplicateSamples)
+                    current_lcLosses['loss'].append(torch.nansum(lossTensor).item()) # Sum of losses across sample for x|=alfa
+    
+                    current_lcLosses['lossTensor'].append(lossTensor)
+                    current_lcLosses['lcSuccesses'].append(lcSuccesses)
+                    current_lcLosses['lcVariables'].append(lcVariables)
+            
+                # Calculate processing time
+                endLC = process_time_ns() # timer()
+                elapsedInNsLC = endLC - startLC
+                elapsedInMsLC = elapsedInNsLC/1000000
+                current_lcLosses['elapsedInMsLC'] += elapsedInMsLC
+
+                if lc.sampleEntries:
+                    self.myLoggerTime.info('Processing time for Lc %s with %i entries and %i variables is: %ims'
+                                           %(lcName, len(lossList), len(lcVariables), current_lcLosses['elapsedInMsLC']))
+                if eliminateDuplicateSamples: 
+                    self.myLoggerTime.info('Processing time for Lc %s with %i entries, %i variables and %i unique samples is: %ims'
+                                           %(lcName, len(lossList), len(lcVariables), lcSampleSize, current_lcLosses['elapsedInMsLC']))
+                else:
+                    self.myLoggerTime.info('Processing time for Lc %s with %i entries and %i variables is: %ims'
+                                           %(lcName, len(lossList), len(lcVariables), current_lcLosses['elapsedInMsLC']))
+        
         self.myLogger.info('')
 
         self.myLogger.info('Processed %i logical constraints'%(lcCounter))
         self.myLoggerTime.info('Processed %i logical constraints'%(lcCounter))
-              
-        if sample: # Calculate global sample loss  
-            globalLoss = 0
-            globalCountFailures = 0
-            
-            for lc in lcLosses:
-                globalLoss += lcLosses[lc]['loss']
-                globalCountFailures += lcLosses[lc]['failuresRate']
-                         
-            lcLosses['globalLoss'] = globalLoss
-            lcLosses['globalFailuresRate'] = globalCountFailures/len(lcLosses)
-                
-            self.myLogger.info('Calculated sample global loss: %f'%(lcLosses['globalLoss']))
-            self.myLoggerTime.info('Calculated sample global loss: %f'%(lcLosses['globalLoss']))
-            self.myLogger.info('Calculated sample global average Failures rate : %f'%(lcLosses['globalFailuresRate']))
-            self.myLoggerTime.info('Calculated sample global average Failures rate: %f'%(lcLosses['globalFailuresRate']))
             
         end = process_time() # timer()
         elapsedInS = end - start
         
-        self.myLogger.info('End Loss Calculation - elapsed time: %is'%(elapsedInS))
-        self.myLoggerTime.info('End Loss Calculation - elapsed time: %is'%(elapsedInS))
+        self.myLogger.info('End Loss Calculation - total time: %is'%(elapsedInS))
+        self.myLoggerTime.info('End Loss Calculation - total time: %is'%(elapsedInS))
         self.myLogger.info('')
         self.myLoggerTime.info('')
 
diff --git a/regr/solver/ilpOntSolver.py b/regr/solver/ilpOntSolver.py
index 42f7b722e..13e5f2b08 100644
--- a/regr/solver/ilpOntSolver.py
+++ b/regr/solver/ilpOntSolver.py
@@ -83,7 +83,11 @@ def setup_solver_logger(self, _ilpConfig = ilpConfig):
         print("Log file for %s is in: %s"%(logName + "Time",chAll.baseFilename))
 
         self.myLogger = logger
+        self.myLogger.info('--- Starting new run ---')
+
         self.myLoggerTime = loggerTime
+        self.myLoggerTime.info('--- Starting new run ---')
+
 
     def loadOntology(self, ontologies):
         start = datetime.datetime.now()
diff --git a/regr/solver/lcLossBooleanMethods.py b/regr/solver/lcLossBooleanMethods.py
index 19879ace2..69bdfb796 100644
--- a/regr/solver/lcLossBooleanMethods.py
+++ b/regr/solver/lcLossBooleanMethods.py
@@ -1,4 +1,5 @@
 import logging
+import torch
 
 from regr.graph import DataNode
 
@@ -10,6 +11,7 @@ class lcLossBooleanMethods(ilpBooleanProcessor):
     def __init__(self, _ildConfig = ilpConfig) -> None:
         super().__init__()
         self.tnorm = DataNode.tnormsDefault
+        self.grad = True
         
         self.myLogger = logging.getLogger(ilpConfig['log_name'])
         self.ifLog =  ilpConfig['ifLog']
@@ -26,19 +28,24 @@ def setTNorm(self, tnorm='L'):
 
         self.tnorm = tnorm
         
-    def _isNumber(self, v):
+    def _isTensor(self, v):
         if v is None:
-            return  False
+            return  0
+        elif torch.is_tensor(v):
+            if len(v.shape) == 0 or len(v.shape) == 1 and v.shape[0] == 1:
+                return v.item()
+            else:
+                0
         elif isinstance(v, (int, float, complex)):
-            return True
+            return 0
         else:
-            return False
+            return 0
         
     def _fixVar(self, var):
         varFixed = []  
         for v in var:
-            if not self._isNumber(v):
-                varFixed.append(0)
+            if v == None or self._isTensor(v) == 0:
+                varFixed.append(torch.zeros(1, device=self.current_device, requires_grad=True))
             else:
                 varFixed.append(v)
         
@@ -50,13 +57,13 @@ def notVar(self, _, var, onlyConstrains = False):
                 
         if self.ifLog: self.myLogger.debug("%s called with : %s"%(logicMethodName,var))
         
-        if not self._isNumber(var):
-            var = 0
+        if not self._isTensor(var):
+            var = torch.zeros(1, device=self.current_device, requires_grad=True)
             
-        notSuccess = 1 - var
+        notSuccess = torch.sub(1, var)
 
         if onlyConstrains:
-            notLoss = 1 - notSuccess
+            notLoss = torch.sub(1, notSuccess)
             
             return notLoss
         else:            
@@ -68,21 +75,22 @@ def and2Var(self, _, var1, var2, onlyConstrains = False):
             
         if self.ifLog: self.myLogger.debug("%s called with: var1 - %s, var2 - %s"%(logicMethodName,var1,var2))
         
-        if not self._isNumber(var1):
+        if not self._isTensor(var1):
             var1 = 0
             
-        if not self._isNumber(var2):
+        if not self._isTensor(var2):
             var2 = 0
             
         if self.tnorm =='L':
-            and2Success = max(0, var1 + var2 - 1)
+            tZero =torch.zeros(1, device=self.current_device, requires_grad=True)
+            and2Success = torch.maximum(tZero, torch.sub(torch.add(var1, var2), 1)) # max(0, var1 + var2 - 1)
         elif self.tnorm =='G':
-            and2Success = min(var1, var2)
+            and2Success = torch.minimum(var1, var2) # min(var1, var2)
         elif self.tnorm =='P':
-            and2Success = var1*var2
+            and2Success = torch.mul(var1, var2) # var1*var2
          
         if onlyConstrains:
-            and2Loss = 1 - and2Success
+            and2Loss = torch.sub(1, and2Success)
             
             return and2Loss
         else:
@@ -95,20 +103,24 @@ def andVar(self, _, *var, onlyConstrains = False):
         if self.ifLog: self.myLogger.debug("%s called with: %s"%(logicMethodName, var))
               
         var = self._fixVar(var)
-                
-        N = len(var)
+                    
+        if self.tnorm =='L':
+            N = len(var)
         
-        varSum = 0
-        for currentVar in var:
-            varSum += currentVar 
+            varSum = torch.clone(var[0])
+            for v in var[1:]:
+                varSum.add_(v)
             
-        if self.tnorm =='L':
-            andSuccess = max(varSum - N + 1, 0)
+            tZero =torch.zeros(1, device=self.current_device, requires_grad=True)
+            andSuccess = torch.maximum(torch.add(torch.sub(varSum,N), 1), tZero) # max(varSum - N + 1, 0)
         elif self.tnorm =='G':
-            andSuccess = min(var)
+            andSuccess = torch.clone(var[0])
+            for v in var[1:]:
+                andSuccess = torch.minimum(andSuccess, v)
         elif self.tnorm =='P':
-            import math
-            andSuccess = math.prod(var)
+            andSuccess = torch.clone(var[0])
+            for v in var[1:]:
+                andSuccess.mul_(v)
 
         if onlyConstrains:
             andLoss = 1 - andSuccess
@@ -123,21 +135,22 @@ def or2Var(self, _, var1, var2, onlyConstrains = False):
        
         if self.ifLog: self.myLogger.debug("%s called with : var1 - %s, var2 - %s"%(logicMethodName,var1,var2))
 
-        if not self._isNumber(var1):
+        if not self._isTensor(var1):
             var1 = 0
             
-        if not self._isNumber(var2):
+        if not self._isTensor(var2):
             var2 = 0
             
         if self.tnorm =='L':
-            or2Success = min(var1 + var2, 1)
+            tOne = torch.ones(1, device=self.current_device, requires_grad=True)
+            or2Success = torch.minimum(torch.add(var1, var2), tOne) # min(var1 + var2, 1)
         elif self.tnorm =='G':
-            or2Success = max(var1, var2)
+            or2Success = torch.maximum(var1, var2) # max(var1, var2)
         elif self.tnorm =='P':
-            or2Success = var1+var2 - var1*var2
+            or2Success = torch.sub(torch.add(var1, var2), torch.mul(var1, var2)) # var1+var2 - var1*var2
 
         if onlyConstrains:
-            or2Loss = 1 - or2Success
+            or2Loss = torch.sub(1, or2Success) # 1 - or2Success
            
             return or2Loss
         else:
@@ -151,22 +164,25 @@ def orVar(self, _, *var, onlyConstrains = False):
         
         var = self._fixVar(var)
         
-        N = len(var)
-        
-        varSum = 0
-        for currentVar in var:            
-            varSum += currentVar
+        varSum = torch.clone(var[0])
+        for v in var[1:]:
+            varSum.add_(v)
            
         if self.tnorm =='L':
-            orSuccess = min(varSum, 1)
+            tOne = torch.ones(1, device=self.current_device, requires_grad=True)
+            orSuccess = torch.minimum(varSum, tOne) # min(varSum, 1)
         elif self.tnorm =='G':
-            orSuccess = max(var)
+            orSuccess = torch.clone(var[0])
+            for v in var[1:]:
+                orSuccess.maximum(v)
         elif self.tnorm =='P':
-            import math
-            orSuccess = varSum - math.prod(var)
+            varPod = torch.clone(var[0])
+            for v in var[1:]:
+                varPod.mul_(v)
+            orSuccess = torch.sub(varSum, varPod) # varSum - math.prod(var)
             
         if onlyConstrains:
-            orLoss = 1 - orSuccess
+            orLoss = torch.sub(1, orSuccess) # 1 - orSuccess
                 
             return orLoss
         else:            
@@ -177,18 +193,12 @@ def nand2Var(self, _, var1, var2, onlyConstrains = False):
         logicMethodName = "NAND"
         
         if self.ifLog: self.myLogger.debug("%s called with: var1 - %s, var2 - %s"%(logicMethodName,var1,var2))
-        
-        if not self._isNumber(var1):
-            var1 = 0
-            
-        if not self._isNumber(var2):
-            var2 = 0
             
         # nand(var1, var2) = not(and(var1, var2))
         nand2Success = self.notVar(_, self.and2Var(_, var1, var2))
 
         if onlyConstrains:
-            nand2Loss = 1 - nand2Success
+            nand2Loss = torch.sub(1, nand2Success)
                         
             return nand2Loss
         else:
@@ -200,21 +210,11 @@ def nandVar(self, _, *var, onlyConstrains = False):
        
         if self.ifLog: self.myLogger.debug("%s called with: %s"%(logicMethodName, var))
         
-        var = self._fixVar(var)
-                
-        nandLoss = 0
-        
-        N = len(var)
-        
-        varSum = 0
-        for currentVar in var:
-            varSum += currentVar
-        
         # nand(var) = not(and(var))
         nandSuccess = self.notVar(_, self.andVar(_, *var))
 
         if onlyConstrains:
-            nandLoss = 1 - nandSuccess
+            nandLoss = torch.sub(1, nandSuccess)
                         
             return nandLoss
         else:            
@@ -226,31 +226,34 @@ def ifVar(self, _, var1, var2, onlyConstrains = False):
 
         if self.ifLog: self.myLogger.debug("%s called with: var1 - %s, var2 - %s"%(logicMethodName,var1,var2))
      
-        if not self._isNumber(var1):
+        var1Item = self._isTensor(var1)
+        if var1Item == None:
             var1 = 0
             
-        if not self._isNumber(var2):
+        var2Item = self._isTensor(var2)
+        if var2Item == None:
             var2 = 0
           
         if self.tnorm =='L':
-            ifSuccess = min(1, 1 - var1 + var2)
+            tOne = torch.ones(1, device=self.current_device, requires_grad=True)
+            ifSuccess = torch.minimum(tOne, torch.add(torch.sub(1, var1), var2)) # min(1, 1 - var1 + var2) #torch.sub
         elif self.tnorm =='G':
-            if var2 > var1: 
-                ifSuccess = 1
+            if var2Item > var1Item: 
+                ifSuccess = torch.mul(var2, torch.div(1, var2)) # 1
             else: 
                 ifSuccess = var2
         elif self.tnorm =='P':
-            if var1 != 0:
-                ifSuccess = min(1, var2/var1)
+            if var1Item != 0:
+                tOne = torch.ones(1, device=self.current_device, requires_grad=True)
+                ifSuccess = torch.minimum(tOne, torch.div(var2, var1)) # min(1, var2/var1) # 
             else:
-                ifSuccess = 1
+                ifSuccess = torch.mul(var2, torch.div(1, var2)) # 1
 
-              
         # if(var1, var2) = or(not(var1), var2)
         #ifSuccess = self.or2Var(_, self.notVar(_, var1), var2)
 
         if onlyConstrains:
-            ifLoss = 1 - ifSuccess
+            ifLoss = torch.sub(1, ifSuccess) # 1 - ifSuccess
             
             return ifLoss
         else:            
@@ -262,17 +265,11 @@ def norVar(self, _, *var, onlyConstrains = False):
         
         if self.ifLog: self.myLogger.debug("%s called with: %s"%(logicMethodName,var))
 
-        var = self._fixVar(var)
-                
-        varSum = 0
-        for currentVar in var:
-            varSum += currentVar
-    
         # nor(var) = not(or(var)
         norSucess = self.notVar(_, self.orVar(_, *var))
         
         if onlyConstrains:
-            norLoss = 1 - norSucess
+            norLoss = torch.sub(1, norSucess)
             
             return norLoss
         else:
@@ -284,17 +281,11 @@ def xorVar(self, _, var1, var2, onlyConstrains = False):
         
         if self.ifLog: self.myLogger.debug("%s called with: var1 - %s, var2 - %s"%(logicMethodName,var1,var2))
 
-        if not self._isNumber(var1):
-            var1 = 0
-            
-        if not self._isNumber(var2):
-            var2 = 0
-            
         # xor(var1, var2) = or(and(var1, not(var2)), and(not(var1), var2))
         xorSuccess = self.or2Var(_, self.and2Var(_, var1, self.notVar(_, var2)), self.and2Var(_, self.notVar(_, var1), var2))
         
         if onlyConstrains:
-            xorLoss = 1 - xorSuccess
+            xorLoss = torch.sub(1, xorSuccess)
             
             return xorLoss
         else:
@@ -305,18 +296,12 @@ def epqVar(self, _, var1, var2, onlyConstrains = False):
         logicMethodName = "EPQ"
         
         if self.ifLog: self.myLogger.debug("%s called with: var1 - %s, var2 - %s"%(logicMethodName,var1,var2))
-
-        if not self._isNumber(var1):
-            var1 = 0
-            
-        if not self._isNumber(var2):
-            var2 = 0
             
         # epq(var1, var2) = and(or(var1, not(var2)), or(not(var1), var2)))
         epqSuccess = self.and2Var(_, self.or2Var(_, var1, self.notVar(_, var2)), self.or2Var(_, self.notVar(_, var1), var2))
         
         if onlyConstrains:
-            epqLoss = 1 - epqSuccess
+            epqLoss = torch.sub(1, epqSuccess)
             
             return epqLoss
         else:
@@ -330,23 +315,23 @@ def countVar(self, _, *var, onlyConstrains = False, limitOp = '==', limit = 1, l
 
         var = self._fixVar(var)
                 
-        varSum = 0
-        for currentVar in var:
-            varSum += currentVar
-            
-        countSuccess = 0
-            
+        varSum = torch.clone(var[0])
+        for v in var[1:]:
+            varSum.add_(v)
+                        
+        tZero = torch.zeros(1, device=self.current_device, requires_grad=True)
+        tOne = torch.ones(1, device=self.current_device, requires_grad=True)
         if limitOp == '>=': # > limit
-            countSuccess = min(max(varSum - limit, 0), 1)
+            countSuccess = torch.minimum(torch.maximum(torch.sub(varSum, limit), tZero), tOne) # min(max(varSum - limit, 0), 1)
             
         elif limitOp == '<=': # < limit
-            countSuccess = min(max(limit - varSum, 0), 1)
+            countSuccess = torch.minimum(torch.maximum(torch.sub(limit, varSum), tZero), tOne) # min(max(limit - varSum, 0), 1)
 
         elif limitOp == '==': # == limit
-            countSuccess = min(max(abs(varSum - limit), 0), 1)
+            countSuccess = torch.minimum(torch.maximum(torch.abs(torch.sub(limit, varSum)), tZero), tOne) # min(max(abs(varSum - limit), 0), 1)
                 
         if onlyConstrains:
-            countLoss = 1 - countSuccess
+            countLoss = torch.sub(1, countSuccess)
     
             return countLoss
         else:
@@ -357,10 +342,10 @@ def fixedVar(self, _, _var, onlyConstrains = False):
         
         if self.ifLog: self.myLogger.debug("%s called with: %s"%(logicMethodName,_var))
         
-        fixedSuccess = 1
+        fixedSuccess = torch.ones(1, device=self.current_device, requires_grad=True)
         
         if onlyConstrains:
-            fixedLoss = 1 - fixedSuccess
+            fixedLoss = torch.sub(1,  fixedSuccess)
     
             return fixedLoss
         else:
diff --git a/regr/solver/lcLossSampleBooleanMethods.py b/regr/solver/lcLossSampleBooleanMethods.py
index f6854ddff..1152d97f5 100644
--- a/regr/solver/lcLossSampleBooleanMethods.py
+++ b/regr/solver/lcLossSampleBooleanMethods.py
@@ -182,7 +182,8 @@ def fixedVar(self, _, _var, onlyConstrains = False):
 class lcLossSampleBooleanMethods(ilpBooleanProcessor):
     def __init__(self, _ildConfig = ilpConfig) -> None:
         super().__init__()
-
+        self.grad = False
+        
         self.myLogger = logging.getLogger(ilpConfig['log_name'])
         self.ifLog = ilpConfig['ifLog']
     
@@ -350,17 +351,12 @@ def countVar(self, _, *var, onlyConstrains = False, limitOp = '==', limit = 1, l
         if self.ifNone(var):
             return None
         
-        limitTensor = torch.full([len(var[0])], limit)
+        limitTensor = torch.full([len(var[0])], limit, device = var[0].device)
        
-        # Translate boolean to int
-        varInt = []
-        for v in var:
-            varInt.append(v.to(dtype=torch.int, copy=True))
-           
         # Calculate sum 
-        varSum = varInt[0]
-        for i in range(1, len(varInt)):
-            varSum += varInt[i]
+        varSum = var[0].int()
+        for i in range(1, len(var)):
+            varSum.add_(var[i].int())
 
         # Check condition
         if limitOp == '>=':
diff --git a/setup.py b/setup.py
new file mode 100644
index 000000000..4b06bf383
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,2 @@
+from setuptools import setup, find_packages
+setup(name="regr", packages=find_packages())
\ No newline at end of file


Test patch:
diff --git a/test_regr/examples/conll04/.gitignore b/test_regr/examples/conll04/.gitignore
new file mode 100644
index 000000000..6d9594aa4
--- /dev/null
+++ b/test_regr/examples/conll04/.gitignore
@@ -0,0 +1,1 @@
+/logs/
diff --git a/test_regr/examples/conll04/test_main.py b/test_regr/examples/conll04/test_main.py
index 4601ed3ca..2dd5351cb 100644
--- a/test_regr/examples/conll04/test_main.py
+++ b/test_regr/examples/conll04/test_main.py
@@ -37,7 +37,7 @@ def test_case():
             'raw': ['John', 'works', 'for', 'IBM'],
             'emb': word_emb,
             #                             John        works       for           IBM
-            'people':       torch.tensor([[0.3, 0.7], [0.9, 0.1], [0.98, 0.02], [0.40, 0.6]], device=device),
+            'people':       torch.tensor([[0.3, 0.7], [0.9, 0.1], [0.98, 0.02], [0.50, 0.5]], device=device),
             'organization': torch.tensor([[0.5, 0.5], [0.8, 0.2], [0.97, 0.03], [0.09, 0.91]], device=device),
             'location':     torch.tensor([[0.7, 0.3], [0.4, 0.6], [0.95, 0.05], [0.50, 0.50]], device=device),
             'other':        torch.tensor([[0.7, 0.3], [0.6, 0.4], [0.90, 0.10], [0.70, 0.30]], device=device),
@@ -114,9 +114,9 @@ def test_case():
         # ifL(work_for('x'), andL(people(path=('x', rel_pair_word1.name)), organization(path=('x', rel_pair_word2.name))))
         #                                 John           works          for      IBM
         'lc2LossTensor' : {"L" : torch.tensor([0.3515,         0.3543,        0.3543,  0.2717, # John
-                                               0,              0,             0.4502,  0.3971, # works
+                                               float("nan"),   float("nan"),  0.4502,  0.3971, # works
                                                0.2769,         0.3385,        0.2891,  0.3100, # for
-                                               0.5246,         0.3543,        0.3100,  0.1572], # IBM
+                                               0.5744,         0.3543,        0.3100,  0.2071], # IBM
                          # torch.tensor([0.2000,         0.2000,        0.2000,  0.0200,  # John
                          #             float("nan"),   float("nan"),  0.4000,  0.2900,  # works
                          #              0.0200,         0.0300,        0.0500,  0.1000,  # for
@@ -128,9 +128,9 @@ def test_case():
                                                0.5000,          0.6457,       0.7191,   0.0000], device=device),
                            
                            "P" : torch.tensor([0.3350,          0.4013,       0.5254,   0.2639, 
-                                               0.0000,          0.0000,       0.8065,   0.4637, 
+                                               float("nan"),   float("nan"),  0.8065,   0.4637, 
                                                0.5000,          0.7102,       0.7309,   0.3800, 
-                                               0.5214,          0.4502,       0.5018,  0.0488], device=device)
+                                               0.5648,          0.5000,       0.5470,  0.0488], device=device)
                            }
     
     }
@@ -370,7 +370,7 @@ def test_main_conll04(case):
         # ------------ Calculate logical constraints losses 
         for tnorm in ['L', 'G', "P"]:
             lcResult = datanode.calculateLcLoss(tnorm=tnorm)
-                    
+                                    
             for i in range(3):
                 assert round(lcResult['LC0']['lossTensor'][i].item(), 4) == round(case.lc0LossTensor[tnorm][i].item(), 4)