utils/json_to_spacy.py

import json
import spacy
from spacy.tokens import DocBin

def read_in_chunks(file_path, chunk_size=1024):
    with open(file_path, 'r', encoding='utf-8') as file:
        while True:
            data = file.read(chunk_size)
            if not data:
                break
            yield data

def convert_json_to_spacy(json_file_path, spacy_file_path):
    # Read the file in chunks and combine the chunks
    file_content = ""
    for chunk in read_in_chunks(json_file_path):
        file_content += chunk

    # Parse the JSON data
    data = json.loads(file_content)

    # Prepare the data for spaCy
    spacy_format = []

    for item in data:
        text = item[0]  # The first element in the list is the text
        entities = item[1]['entities']  # The second element contains the dictionary with 'entities'
        spacy_entities = [(start, end, label) for start, end, label in entities]
        spacy_format.append({"text": text, "entities": spacy_entities})

    # Create a blank English model
    nlp = spacy.blank("en")

    # Initialize a DocBin object
    doc_bin = DocBin()

    # Convert the data to spaCy Doc objects and add to DocBin
    for entry in spacy_format:
        doc = nlp.make_doc(entry["text"])
        # Convert entities
        entities = []
        seen_positions = set()  # To track positions and avoid overlap
        for start, end, label in entry["entities"]:
            # Ensure span is within the document's length
            if start < 0 or end > len(doc.text) or start >= end:
                print(f"Invalid span: start={start}, end={end}, label={label}")
                continue

            # Check for overlaps and prioritize entities
            if not any(start < e_end and end > e_start for e_start, e_end, _ in seen_positions):
                span = doc.char_span(start, end, label=label)
                if span is not None:
                    entities.append(span)
                    seen_positions.add((start, end, label))
            else:
                print(f"Overlapping span: start={start}, end={end}, label={label}")
        
        # Set entities
        doc.ents = entities
        
        # Add to DocBin
        doc_bin.add(doc)

    # Save the DocBin to a .spacy file
    doc_bin.to_disk(spacy_file_path)

    print(f"Data has been successfully saved to {spacy_file_path}!")