New matching logic for grader

data/data_loader.py

@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 DATA_DIR = os.path.join(os.path.dirname(__file__))
 DEFAULT_CONTRACTS_FILE = os.path.join(DATA_DIR, "contracts.json")
-
+DEFAULT_VUNERABILITIES_FILE = os.path.join(DATA_DIR, "vulnerabilities.json")
 
 def load_contracts(path: str = DEFAULT_CONTRACTS_FILE) -> List[Dict[str, Any]]:
     """Load and return all contracts from the JSON dataset."""
@@ -22,6 +22,12 @@ def load_contracts(path: str = DEFAULT_CONTRACTS_FILE) -> List[Dict[str, Any]]:
         return json.load(f)
 
 
+def load_vulnerabilities(path: str = DEFAULT_VUNERABILITIES_FILE) -> List[Dict[str, Any]]:
+    """Load and return all vulnerability entries from the JSON dataset."""
+    with open(path, "r") as f:
+        return json.load(f)
+
+
 def get_all_vulnerable_entries(
     contracts: List[Dict[str, Any]],
 ) -> List[Tuple[Dict[str, Any], Dict[str, Any]]]:

data/vulnerabilities.json

@@ -0,0 +1,192 @@
+[
+  {
+    "vulnerability": "Non-compliance with EIP4626 standard - previewDeposit",
+    "terms": [
+      "previewdeposit non-compliance",
+      "eip4626 previewdeposit issue",
+      "previewdeposit violation",
+      "previewdeposit standard break",
+      "previewdeposit limitation include",
+      "eip-4626 previewdeposit violation",
+      "previewdeposit spec deviation",
+      "previewdeposit max limitation error",
+      "4626 compliance failure (previewdeposit)",
+      "eip4626",
+      "previewdeposit",
+      "compliance",
+      "standard violation",
+      "maxdeposit",
+      "limitation"
+    ]
+  },
+  {
+    "vulnerability": "Additive burn (rounding vulnerability)",
+    "terms": [
+      "additive burn",
+      "rounding burn",
+      "burn rounding",
+      "withdraw rounding zero burn",
+      "atoken burn rounding",
+      "rounding burn attack",
+      "additive burn rounding",
+      "withdraw rounding to zero",
+      "atoken conversion rounding",
+      "burn rounding vulnerability",
+      "rounding",
+      "burn",
+      "atoken",
+      "withdraw",
+      "precision loss",
+      "conversion rate",
+      "zero burn"
+    ]
+  },
+  {
+    "vulnerability": "Additive mint (Stable debt token)",
+    "terms": [
+      "additive mint",
+      "stable debt rounding",
+      "mint rounding vulnerability",
+      "debt token rounding",
+      "deposit rounding mint",
+      "rounding mint attack",
+      "additive mint rounding",
+      "stable debt token rounding",
+      "deposit rounding mint",
+      "debt token inflation",
+      "rounding",
+      "mint",
+      "stable debt",
+      "deposit",
+      "precision loss",
+      "debt token",
+      "inflation"
+    ]
+  },
+  {
+    "vulnerability": "Non-compliance with EIP4626 standard - previewMint",
+    "terms": [
+      "previewmint non-compliance",
+      "eip4626 previewmint issue",
+      "previewmint violation",
+      "previewmint standard break",
+      "eip-4626 previewmint violation",
+      "previewmint spec deviation",
+      "previewmint max limitation error",
+      "4626 compliance failure (previewmint)",
+      "eip4626",
+      "previewmint",
+      "compliance",
+      "standard violation",
+      "maxmint",
+      "limitation"
+    ]
+  },
+  {
+    "vulnerability": "Non-compliance with EIP4626 standard - previewWithdraw",
+    "terms": [
+      "previewwithdraw non-compliance",
+      "eip4626 previewwithdraw issue",
+      "previewwithdraw violation",
+      "previewwithdraw standard break",
+      "eip-4626 previewwithdraw violation",
+      "previewwithdraw spec deviation",
+      "previewwithdraw max limitation error",
+      "4626 compliance failure (previewwithdraw)",
+      "eip4626",
+      "previewwithdraw",
+      "compliance",
+      "standard violation",
+      "maxwithdraw",
+      "limitation"
+    ]
+  },
+  {
+    "vulnerability": "Non-compliance with EIP4626 standard - previewRedeem",
+    "terms": [
+      "previewredeem non-compliance",
+      "eip4626 previewredeem issue",
+      "previewredeem violation",
+      "previewredeem standard break",
+      "eip-4626 previewredeem violation",
+      "previewredeem spec deviation",
+      "previewredeem max limitation error",
+      "4626 compliance failure (previewredeem)",
+      "eip4626",
+      "previewredeem",
+      "compliance",
+      "standard violation",
+      "maxredeem",
+      "limitation"
+    ]
+  },
+  {
+    "vulnerability": "Non-compliance with EIP4626 standard - non-reverting functions",
+    "terms": [
+      "non-reverting functions revert",
+      "eip4626 revert issue",
+      "totalassets revert",
+      "max functions revert",
+      "arithmetic revert vulnerability",
+      "eip-4626 reverting view functions",
+      "totalassets revert",
+      "max functions revert",
+      "arithmetic revert in 4626",
+      "view function revert violation",
+      "eip4626",
+      "revert",
+      "totalassets",
+      "maxdeposit",
+      "maxmint",
+      "maxwithdraw",
+      "maxredeem",
+      "arithmetic underflow",
+      "overflow"
+    ]
+  },
+  {
+    "vulnerability": "Discount Factor Issue (Unfair finalization within a batch)",
+    "terms": [
+      "discount factor unfair",
+      "batch finalization unfair",
+      "fifo violation",
+      "discount factor flaw",
+      "unfair batch finalization",
+      "fifo fairness violation",
+      "batch finalization unfairness",
+      "discount factor miscalculation",
+      "queue finalization manipulation",
+      "unfair slashing distribution",
+      "discount factor",
+      "finalization",
+      "batch",
+      "fifo",
+      "fairness",
+      "queue",
+      "slashing",
+      "share rate"
+    ]
+  },
+  {
+    "vulnerability": "Potential for incorrect ETH transfer due to price calculation",
+    "terms": [
+      "incorrect eth transfer",
+      "price calculation error",
+      "claim eth wrong",
+      "discounted batch error",
+      "eth transfer miscalculation",
+      "claim amount miscalculation",
+      "eth transfer error",
+      "price calculation flaw",
+      "discounted batch eth error",
+      "incorrect eth withdrawal",
+      "eth transfer",
+      "claim",
+      "price calculation",
+      "discounted batch",
+      "finalization",
+      "incorrect amount",
+      "fund loss"
+    ]
+  }
+]

data/vulnerabilities.md

@@ -0,0 +1,31 @@
+First Line is always the issue and second line is the issue description then space and the same repeats.
+
+Non-compliance with EIP4626 standard - previewDeposit
+As per EIP4626, all the preview functions must not take into account any limitation of the system, like those returned by the max() methods. In the contract, the preview methods do take into account system limitations.
+
+Additive burn (rounding vulnerability)
+Due to rounding in conversions to AToken, if the conversion rate is high enough, one can withdraw a small amount that will result in the system transferring underlying tokens but burning zero ATokens of the user's account
+
+Additive mint (Stable debt token)
+Due to rounding in conversions to stable debt token, if the conversion rate is high enough, one can deposit a small amount that will result in the system transferring underlying tokens but minting debt tokens to the user's account
+
+Non-compliance with EIP4626 standard - previewDeposit
+As per EIP4626, all the preview functions must not take into account any limitation of the system, like those returned by the max() methods. In the contract, the preview methods do take into account system limitations.
+
+Non-compliance with EIP4626 standard - previewMint
+As per EIP4626, all the preview functions must not take into account any limitation of the system, like those returned by the max() methods. In the contract, the preview methods do take into account system limitations.
+
+Non-compliance with EIP4626 standard - previewWithdraw
+As per EIP4626, all the preview functions must not take into account any limitation of the system, like those returned by the max() methods. In the contract, the preview methods do take into account system limitations
+
+Non-compliance with EIP4626 standard - previewRedeem
+As per EIP4626, all the preview functions must not take into account any limitation of the system, like those returned by the max() methods. In the contract, the preview methods do take into account system limitations
+
+Non-compliance with EIP4626 standard - non-reverting functions
+as per EIP4626, the functions totalAssets, maxDeposit, maxMint, maxWithdraw, and maxRedeem must not revert by any means. In the contract, however, these functions may revert due to over/underflows of arithmetical computations
+
+Discount Factor Issue (Unfair finalization within a batch)
+If two users in the same finalization batch entered the queue at different share rates (e.g., before and after a slashing event), the finalization logic weights the amount incorrectly. This can lead to one user losing ETH at the expense of another user within the same batch, which violates the FIFO principle of fairness.
+
+Potential for incorrect ETH transfer due to price calculation
+The `claim` function relies on the `_calculateDiscountedBatch` function, which was part of the original discount factor logic. The flaw in the `finalize` function's discount factor calculation directly impacts the amount of ETH that a user can claim. If the finalization was processed with a flawed discount factor, the user will claim an incorrect amount of ETH, either losing funds or gaining more than entitled

eval.py

@@ -82,6 +82,7 @@ def oracle_agent(env: Task1Environment, seed: int, verbose: bool = False) -> Dic
     for contract in contracts:
         for fn in contract.get("functions", []):
             if fn["name"].lower() == target_fn.lower() and fn.get("vulnerable"):
+                # ! SINCE OUR MATCHER IS BASED ON FACT THAT EXPECTED STRING IS 2-3 WORDS, THIS DOESN'T MATCH WELL
                 vuln_issue = fn["vulnerability_details"]["issue"]
                 break
         if vuln_issue:

tasks/task1/environment.py

@@ -296,8 +296,8 @@ class Task1Environment(BaseEnv):
                     "Submit requires 'function_name' and 'vulnerability_type' in params.",
                     Reward(value=-0.5, reason="Malformed submission", partial=True),
                 )
-            score = self._grader.grade_submission(fn_name, vuln_type)
-            reward_val = self._grader.reward_for_score(score)
+            score = self._grader.grade_submission(fn_name, vuln_type) # type: ignore
+            reward_val = self._grader.reward_for_score(score) # type: ignore
             self._done = True
 
             if score == 1.0:
@@ -311,7 +311,7 @@ class Task1Environment(BaseEnv):
                     f"'{vuln_type}' was not precise. Score: 0.5"
                 )
             else:
-                correct = self._grader.get_canonical_answer()
+                correct = self._grader.get_canonical_answer() # type: ignore
                 msg = (
                     f"❌ INCORRECT. '{fn_name}' is not the target vulnerable function. "
                     f"Correct answer: {correct['function']} ({correct['vulnerability']}). Score: 0.0"

tasks/task1/grader.py

@@ -9,75 +9,15 @@ Deterministic grader. Score range: 0.0 – 1.0
 """
 from __future__ import annotations
 from typing import Dict, List, Optional
+from utils.matcher import match_strings
+from data.data_loader import load_vulnerabilities
 
-VULN_KEYWORDS: Dict[str, List[str]] = {
-    "reentrancy": [
-        "reentrancy", "re-entrancy", "reentrant", "re entrant",
-        "recursive call", "reentr",
-    ],
-    "missing access control": [
-        "access control", "missing access", "no access", "unauthorized",
-        "privilege", "permission", "onlyowner", "only owner",
-        "no modifier", "missing modifier", "no check", "anyone can call",
-    ],
-    "integer overflow": [
-        "overflow", "integer overflow", "arithmetic overflow",
-        "safemath", "safe math", "uint overflow", "wraparound",
-        "integer underflow", "underflow",
-    ],
-    "tx.origin authentication": [
-        "tx.origin", "txorigin", "tx origin", "phishing",
-        "origin authentication", "origin auth",
-    ],
-    "front-running": [
-        "front-running", "frontrunning", "front running", "mev",
-        "sandwich", "mempool", "commit reveal", "commit-reveal",
-        "gas price manipulation",
-    ],
-    "timestamp dependence": [
-        "timestamp", "block.timestamp", "time manipulation",
-        "miner timestamp", "time dependency", "timestamp dependence",
-    ],
-    "denial of service": [
-        "denial of service", " dos", "gas limit", "unbounded loop",
-        "block gas", " oog", "out of gas", "infinite loop", "unbounded array",
-        "gas exhaustion",
-    ],
-    "unchecked return value": [
-        "unchecked return", "return value", "unchecked transfer",
-        "silent failure", "safeerc20", "safe transfer", "ignored return",
-        "erc20 return",
-    ],
-}
-
-
-def _norm(text: str) -> str:
-    return text.strip().lower()
-
-
-def _find_bucket(ground_truth_issue: str) -> Optional[str]:
-    """
-    Longest-match keyword search to identify canonical vulnerability bucket.
-    Longest match avoids short-keyword collisions (e.g. 'auth' in 'tx.origin authentication').
-    """
-    norm_gt = _norm(ground_truth_issue)
-    best: Optional[str] = None
-    best_len: int = 0
-    for canonical, keywords in VULN_KEYWORDS.items():
-        for kw in keywords:
-            if kw in norm_gt and len(kw) > best_len:
-                best_len = len(kw)
-                best = canonical
-    return best
-
-
-def match_vuln_keyword(submitted: str, ground_truth_issue: str) -> bool:
-    bucket = _find_bucket(ground_truth_issue)
-    if bucket is None:
-        return _norm(submitted) in _norm(ground_truth_issue)
-    norm_sub = _norm(submitted)
-    return any(kw in norm_sub for kw in VULN_KEYWORDS[bucket])
-
+def match_vuln_keywords(submitted: str, expected: str) -> bool:
+    """Checks if the submitted vulnerability type matches the expected one using keyword matching."""
+    for types in load_vulnerabilities():
+        if types["vulnerability"] == expected:
+            return match_strings(types["terms"], submitted)
+    return False
 
 class Task1Grader:
     def __init__(self, target_function: str, vulnerability_issue: str) -> None:
@@ -87,7 +27,7 @@ class Task1Grader:
     def grade_submission(self, submitted_function: str, submitted_vuln_type: str) -> float:
         if submitted_function.strip().lower() != self.target_function:
             return 0.0
-        return 1.0 if match_vuln_keyword(submitted_vuln_type, self.vulnerability_issue) else 0.5
+        return 1.0 if match_vuln_keywords(submitted_vuln_type, self.vulnerability_issue) else 0.5
 
     def reward_for_score(self, score: float) -> float:
         if score == 1.0: return 5.0

utils/__init__.py

@@ -1 +1,4 @@
 # utils package
+from utils.matcher import match_strings
+
+_all__ = ["match_strings"]

utils/matcher.py

@@ -0,0 +1,29 @@
+from typing import List
+
+def match_strings(terms: List[str], phrase: str, threshold: float = 0.6) -> bool:
+    """
+    Determine if a short string (2-3 words) is likely present in a list of terms
+    using token set similarity
+
+    Args:
+        terms: List of term strings (preferably lowercased).
+        phrase: Input phrase (2-3 words, case-insensitive).
+        threshold: Minimum Jaccard similarity to count as a match (default 0.6).
+
+    Returns:
+        True if any term in the list has token set similarity >= threshold.
+    """
+    phrase_lower = phrase.lower().strip()
+    phrase_tokens = set(phrase_lower.split())
+
+    for term in terms:
+        term_tokens = set(term.split())
+        # Jaccard similarity = size of intersection / size of union
+        if not phrase_tokens or not term_tokens:
+            continue
+        intersection = phrase_tokens & term_tokens
+        union = phrase_tokens | term_tokens
+        similarity = len(intersection) / len(union)
+        if similarity >= threshold:
+            return True
+    return False