Spaces:
Running on Zero
Running on Zero
| import re | |
| from typing import Any, Dict, List, Tuple | |
| import unicodedata | |
| SPAN_RE = re.compile(r"<span>(.*?)</span>") | |
| _ALLOWED_EMOJI_GLUE = { | |
| "\u200d", # ZWJ | |
| "\ufe0f", # variation selector-16 | |
| "\u20e3", # combining enclosing keycap | |
| } | |
| def validate_emoji_only(value: str) -> Tuple[bool, str]: | |
| if not isinstance(value, str) or not value.strip(): | |
| return False, "empty emoji string" | |
| s = value.strip() | |
| for ch in s: | |
| if unicodedata.category(ch).startswith("L"): | |
| return False, f"contains letters: {repr(ch)}" | |
| try: | |
| import emoji | |
| except ImportError: | |
| return False, "missing dependency: pip install emoji" | |
| tokens = emoji.emoji_list(s) | |
| if not tokens: | |
| return False, "no emoji found" | |
| covered = [False] * len(s) | |
| for t in tokens: | |
| start, end = t["match_start"], t["match_end"] | |
| for i in range(start, end): | |
| covered[i] = True | |
| for i, ch in enumerate(s): | |
| if covered[i]: | |
| continue | |
| if ch in _ALLOWED_EMOJI_GLUE: | |
| continue | |
| return False, f"contains non-emoji character: {repr(ch)}" | |
| return True, "ok" | |
| def extract_marked_spans(marked_sentence: str) -> List[str]: | |
| return [s.strip() for s in SPAN_RE.findall(marked_sentence)] | |
| def validate_marked_matches_original(marked: str, original: str) -> Tuple[bool, str]: | |
| stripped = marked.replace("<span>", "").replace("</span>", "") | |
| if stripped != original: | |
| # Find first differing character for diagnosis | |
| for i, (a, b) in enumerate(zip(stripped, original)): | |
| if a != b: | |
| ctx_s = repr(stripped[max(0, i-10):i+10]) | |
| ctx_o = repr(original[max(0, i-10):i+10]) | |
| return False, f"char {i}: got {ctx_s}, expected {ctx_o}" | |
| # One is a prefix of the other | |
| shorter = "stripped" if len(stripped) < len(original) else "original" | |
| return False, f"{shorter} is shorter ({len(stripped)} vs {len(original)} chars)" | |
| return True, "ok" | |
| def validate_marking_json(obj: Dict[str, Any], original_sentence: str) -> Tuple[bool, str]: | |
| if not isinstance(obj, dict): | |
| return False, "marking result is not a dict" | |
| marked = obj.get("marked") | |
| if not isinstance(marked, str): | |
| return False, "missing/invalid 'marked' field" | |
| return validate_marked_matches_original(marked, original_sentence) | |
| def validate_indexed_annotation_json(obj: Dict[str, Any], original: str) -> Tuple[bool, str]: | |
| """Validate an indexed-format annotation dict against the original sentence. | |
| Expected shape: {"annotations": [{"start": int, "end": int, "text": str, "emojis": str}, ...]} | |
| Checks: valid list, each entry has int start/end within bounds, text matches original[start:end], | |
| emoji-only content, and spans are ordered left-to-right (non-overlapping). | |
| """ | |
| if not isinstance(obj, dict): | |
| return False, "annotation is not a dict" | |
| annotations = obj.get("annotations") | |
| if not isinstance(annotations, list): | |
| return False, "missing/invalid 'annotations' field" | |
| if not annotations: | |
| return True, "ok" | |
| prev_end = 0 | |
| for i, item in enumerate(annotations): | |
| if not isinstance(item, dict): | |
| return False, f"annotation entry {i} is not a dict" | |
| start = item.get("start") | |
| end = item.get("end") | |
| text = item.get("text") | |
| emojis = item.get("emojis") | |
| if not isinstance(start, int) or not isinstance(end, int): | |
| return False, f"annotation entry {i}: start/end must be integers" | |
| if not (0 <= start < end <= len(original)): | |
| return False, ( | |
| f"annotation entry {i}: offsets [{start}:{end}] out of range " | |
| f"for sentence of length {len(original)}" | |
| ) | |
| if start < prev_end: | |
| return False, ( | |
| f"annotation entry {i}: spans must be ordered and non-overlapping " | |
| f"(start={start} < prev_end={prev_end})" | |
| ) | |
| if not isinstance(text, str): | |
| return False, f"annotation entry {i}: missing/invalid 'text' field" | |
| expected_text = original[start:end] | |
| if text != expected_text: | |
| return False, ( | |
| f"annotation entry {i}: text {text!r} does not match " | |
| f"original[{start}:{end}]={expected_text!r}" | |
| ) | |
| if not isinstance(emojis, str) or not emojis.strip(): | |
| return False, f"annotation entry {i}: empty/missing 'emojis' for span '{text}'" | |
| ok_emoji, why = validate_emoji_only(emojis.strip()) | |
| if not ok_emoji: | |
| return False, f"non-emoji content for span '{text}': {why}" | |
| prev_end = end | |
| return True, "ok" | |
| def validate_annotation_json(obj: Dict[str, Any], original_sentence: str) -> Tuple[bool, str]: | |
| if not isinstance(obj, dict): | |
| return False, "annotation is not a dict" | |
| marked = obj.get("marked") | |
| if not isinstance(marked, str): | |
| return False, "missing/invalid 'marked' field" | |
| annotations = obj.get("annotations") | |
| if not isinstance(annotations, list): | |
| return False, "missing/invalid 'annotations' field" | |
| # Check that stripping tags recovers the original sentence | |
| ok, reason = validate_marked_matches_original(marked, original_sentence) | |
| if not ok: | |
| return False, reason | |
| # Extract spans positionally from marked text (one entry per occurrence, in order) | |
| marked_spans = extract_marked_spans(marked) | |
| # Zero spans is valid (e.g. all stopwords or named entities) | |
| if not marked_spans: | |
| if annotations: | |
| return False, "annotations list is non-empty but no spans found in marked text" | |
| return True, "ok" | |
| # annotations must be a positional list matching marked_spans exactly | |
| if len(annotations) != len(marked_spans): | |
| return False, ( | |
| f"annotations length ({len(annotations)}) != " | |
| f"number of marked spans ({len(marked_spans)})" | |
| ) | |
| for i, (item, expected_span) in enumerate(zip(annotations, marked_spans)): | |
| if not isinstance(item, dict): | |
| return False, f"annotation entry {i} is not a dict" | |
| span = item.get("span") | |
| emojis = item.get("emojis") | |
| if not isinstance(span, str) or not span.strip(): | |
| return False, f"annotation entry {i} has empty/missing 'span'" | |
| if span.strip() != expected_span: | |
| return False, ( | |
| f"annotation entry {i} span {span.strip()!r} " | |
| f"does not match marked span {expected_span!r}" | |
| ) | |
| if not isinstance(emojis, str) or not emojis.strip(): | |
| return False, f"annotation entry {i} has empty/missing 'emojis' for span: {span}" | |
| ok_emoji, why = validate_emoji_only(emojis.strip()) | |
| if not ok_emoji: | |
| return False, f"non-emoji content for span '{span}': {why}" | |
| return True, "ok" | |