serialized_file_creation_demo/utils.py

## This script will take a simplified data model and convert it to a Gen3-style data model

import json
import os
import sys
import shutil
import copy
import random
import datetime
from pathlib import Path
import pandas as pd
import hashlib

###################################################################################################################################################
### PSEUDO CODE
# 1. Read the simplified data model from a JSON file.
# 2. Create a new dictionary to hold the Gen3-style data model.
# 3. Add a generic program, CMC nodes, _terms, _settings, _definitions dict (reference https://github.com/uc-cdis/pypfb/blob/master/examples/minimal-pfb/minimal_file.json to see what's required)
# 4. For each node in the simplified data model:
#    - Convert the node to the Gen3 format.
#    - Add it to the new dictionary.
# 5. Write the new dictionary to a JSON file.
###################################################################################################################################################


######## SCRIPTS


def read_schema(schema):
    """Reads in a schema.json file for a simplified / Gen3 data dictionary and returns a Python dictionary."""
    with open(schema) as json_file:
        try:
            dm = json.load(json_file)
        except json.JSONDecodeError:
            print(f"Error reading schema file: {schema}")
            return {}
    return dm


# # read in the simplified data model
# sdm_file = "./Documents/Notes/AI/AI_data_curation/SDC/sdc_v3_nmax1_nmin1_pmax75_pmin25_limit20_dmax1000_20250423/SDM_0_20250423_tsvs/actionable_mutation/SDM_0__actionable_mutation__jsonschema_dd.json"
# sdm = read_schema(schema=sdm_file)


def write_schema(data, out_file):
    with open(out_file, "w") as f:
        json.dump(data, f, indent=4)
    print(f"\tData model written to {out_file}")


def create_gen3_dm_scaffold():
    """Creates a minimal Gen3 data model scaffold with the required nodes and properties.
    Returns a dictionary representing the scaffold.
        '_definitions',
        '_settings',
        '_terms',
        'core_metadata_collection',
        'data_release',
        'metaschema',
        'program',
        'root']
    """
    scaffold = {}
    ## Add the _terms, _definitions, _settings using the minimal file in pypfb
    mdfile = "minimal_file.json"
    md = read_schema(mdfile)
    gdm = copy.deepcopy(
        {n: md[n] for n in md if n in ["_definitions", "_settings", "_terms"]}
    )
    ########### read in MIDRC data model to create node templates for program, project, and CMC
    ## Add program/project/CMC node
    midrc_dm = "./Documents/Notes/AI/AI_data_curation/input_schemas/gen3_schemas/data.midrc.org_schema.json"
    mdm = read_schema(midrc_dm)
    keep_nodes = [
        "data_release",
        "metaschema",
        "program",
        "project",
        "core_metadata_collection",
        "root",
    ]
    gdm = gdm | copy.deepcopy(
        {n: mdm[n] for n in mdm if n in keep_nodes}
    )  # start with the program node
    gdm["data_release"]["namespace"] = "https://gen3.org"
    gdm["core_metadata_collection"]["namespace"] = "https://gen3.org"
    del gdm["core_metadata_collection"]["properties"]["case_ids"]
    gdm["metaschema"]["properties"]["links"][
        "title"
    ] = "Define a link to other GDC entities"  # remove the title from the links property
    # write the minimal dict scaffold to a file:
    out_dir = "./Gen3/dd/synthetic_data_for_AI"
    out_file = os.path.join(out_dir, "gen3_dm_scaffold.json")
    write_schema(gdm, out_file)
    # read in the minimal Gen3 data model scaffold
    scaffold_file = "./Gen3/dd/synthetic_data_for_AI/gen3_dm_scaffold.json"
    scaffold = read_schema(scaffold_file)
    return scaffold


def convert_sdm_node_to_gen3(node, sdm):
    """Convert a simplified data model node to a Gen3-style data model node dict with these keys:
    ['$schema',
    'additionalProperties',
    'category',
    'description',
    'id',
    'links',
    'namespace',
    'nodeTerms',
    'program',
    'project',
    'properties',
    'required',
    'submittable',
    'systemProperties',
    'title',
    'type',
    'uniqueKeys',
    'validators']
    """
    node_def = [n for n in sdm["nodes"] if n["name"] == node][
        0
    ]  # get the node definition from the simplified data model
    gen3_node = {}
    gen3_node["$schema"] = "http://json-schema.org/draft-04/schema#"
    gen3_node["id"] = node_def.get("name", node)
    gen3_node["title"] = node_def.get(
        "name", node
    )  # TODO: original title is lost; use the node name as the title
    gen3_node["type"] = "object"  # all object
    gen3_node["nodeTerms"] = None
    gen3_node["namespace"] = "https://gen3.org"
    gen3_node["category"] = node_def.get(
        "category", "default"
    )  # # TODO: original category lost, get category from a Gen3 model
    gen3_node["program"] = "*"
    gen3_node["project"] = "*"
    gen3_node["description"] = node_def.get("description", "")
    gen3_node["additionalProperties"] = False
    gen3_node["submittable"] = True
    gen3_node["validators"] = None
    gen3_node["systemProperties"] = [
        "id",
        "project_id",
        "state",
        "created_datetime",
        "updated_datetime",
    ]  # TODO: if it's a file node, should add "file_state", "error_type"
    gen3_node["links"] = []
    for link in node_def.get("links", []):
        if any([l.strip(".id") for l in node_def["required"] if l.startswith(link)]):
            required = True
        else:
            required = False
        if any(
            [
                p.strip(".id")
                for p in [p["name"] for p in node_def["properties"]]
                if p.startswith(link)
            ]
        ):
            name = link
        else:
            name = link
        gen3_node["links"].append(
            {
                "backref": node_def.get("name", node),  # TODO: backref is lost
                "label": "child_of",  # TODO: original label is lost; use 'child_of' as a default
                "multiplicity": "many_to_many",  # TODO: original multiplicity lost; get multiplicity from a Gen3 model
                "name": name,  # TODO: original name is lost; use the link target as the name
                "required": required,
                "target_type": link,
            }
        )
    gen3_node["required"] = node_def.get("required", [])
    gen3_node["uniqueKeys"] = [
        ["id"],
        ["project_id", "submitter_id"],
    ]  # TODO: original uniqueKeys lost; same for all nodes
    gen3_node["properties"] = {}
    # Add properties to the Gen3 node
    for prop_def in node_def.get("properties", []):
        prop = prop_def["name"]
        if prop.endswith(
            ".id"
        ):  # it's a link property, so add all the things for links
            prop = prop[:-3]  # remove the '.id' suffix
            gen3_node["properties"][prop] = {}
            gen3_node["properties"][prop]["description"] = prop_def.get(
                "description", ""
            )
            gen3_node["properties"][prop]["anyOf"] = [
                {
                    "items": {
                        "additionalProperties": True,
                        "maxItems": 1,
                        "minItems": 1,
                        "properties": {
                            "id": {
                                "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
                                "term": {
                                    "description": "A 128-bit identifier. Depending on the mechanism used to generate it, it is either guaranteed to be different from all other UUIDs/GUIDs generated until 3400 AD or extremely likely to be different. Its relatively small size lends itself well to sorting, ordering, and hashing of all sorts, storing in databases, simple allocation, and ease of programming in general.\n",
                                    "termDef": {
                                        "cde_id": "C54100",
                                        "cde_version": None,
                                        "source": "NCIt",
                                        "term": "Universally Unique Identifier",
                                        "term_url": "https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&version=16.02d&ns=NCI_Thesaurus&code=C54100",
                                    },
                                },
                                "type": "string",
                            },
                            "submitter_id": {"type": "string"},
                        },
                        "type": "object",
                    },
                    "type": "array",
                },
                {
                    "additionalProperties": True,
                    "properties": {
                        "id": {
                            "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
                            "term": {
                                "description": "A 128-bit identifier. Depending on the mechanism used to generate it, it is either guaranteed to be different from all other UUIDs/GUIDs generated until 3400 AD or extremely likely to be different. Its relatively small size lends itself well to sorting, ordering, and hashing of all sorts, storing in databases, simple allocation, and ease of programming in general.\n",
                                "termDef": {
                                    "cde_id": "C54100",
                                    "cde_version": None,
                                    "source": "NCIt",
                                    "term": "Universally Unique Identifier",
                                    "term_url": "https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&version=16.02d&ns=NCI_Thesaurus&code=C54100",
                                },
                            },
                            "type": "string",
                        },
                        "submitter_id": {"type": "string"},
                    },
                    "type": "object",
                },
            ]
        else:
            gen3_node["properties"][prop] = {}
            gen3_node["properties"][prop]["description"] = prop_def.get(
                "description", ""
            )
            if "type" in prop_def and prop_def["type"] == "enum":
                prop_type = "string"  # TODO: if the type is enum, we need to specify the enum values, which is lost in simplified data models
            elif "type" in prop_def and prop_def["type"] == "array":
                prop_type = "array"  # TODO: if the type is array, we need to specify items type, which is lost in simplified data models
                gen3_node["properties"][prop]["items"] = {
                    "type": "string"
                }  # default to string if not specified
            else:
                prop_type = prop_def.get(
                    "type", "string"
                )  # default to string if not specified
            gen3_node["properties"][prop]["type"] = prop_type

    return gen3_node


# convert_sdm_node_to_gen3('case',sdm)


def sdm_to_gen3(sdm):
    """Converts nodes in a simplified data model into the Gen3-style data model nodes.
    Returns the Gen3-ified data model as a dictionary.
    """
    gdm = {}
    sdm_nodes = [
        n["name"] for n in sdm["nodes"]
    ]  # get the list of node names from the simplified data model
    for node in sdm_nodes:
        gen3_node = convert_sdm_node_to_gen3(node, sdm)
        gdm[node] = gen3_node
    return gdm


# gdm = sdm_to_gen3(sdm)  # convert simplified data model nodes into the Gen3-style nodes


# merge the scaffold with the Gen3-style data model
def merge_scaffold_into_gdm(gdm, scaffold):
    """Merges the scaffold into the Gen3-style data model.
    The scaffold contains the _terms, _definitions, _settings, program, project, and CMC nodes.
    """
    # Add the scaffold nodes to the Gen3-style data model
    for node in scaffold:
        if node not in gdm:
            gdm[node] = scaffold[node]
    return gdm


# gdm = merge_scaffold_into_gdm(gdm, scaffold)  # merge the scaffold into the Gen3-style data model


def fix_project(gdm):
    """Ensures that the project node links to the program node in the Gen3-style data model.
    Adds a 'program' property to the project node if it doesn't already exist.
    Adds required project node properties if they are missing.
    """
    if "project" in gdm and "program" in gdm:
        # add link to programs
        if "links" in gdm["project"] and not any(
            link["target_type"] == "program" for link in gdm["project"]["links"]
        ):
            gdm["project"]["links"].append(
                {
                    "backref": "project",
                    "label": "member_of",
                    "multiplicity": "many_to_one",
                    "name": "program",
                    "required": True,
                    "target_type": "program",
                }
            )
        # add link name to properties
        if (
            "properties" in gdm["project"]
            and "program" not in gdm["project"]["properties"]
        ):
            gdm["project"]["properties"]["program"] = {
                "anyOf": [
                    {
                        "items": {
                            "additionalProperties": True,
                            "maxItems": 1,
                            "minItems": 1,
                            "properties": {
                                "id": {
                                    "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
                                    "term": {
                                        "description": "A 128-bit identifier. Depending on the mechanism used to generate it, it is either guaranteed to be different from all other UUIDs/GUIDs generated until 3400 AD or extremely likely to be different. Its relatively small size lends itself well to sorting, ordering, and hashing of all sorts, storing in databases, simple allocation, and ease of programming in general.\n",
                                        "termDef": {
                                            "cde_id": "C54100",
                                            "cde_version": None,
                                            "source": "NCIt",
                                            "term": "Universally Unique Identifier",
                                            "term_url": "https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&version=16.02d&ns=NCI_Thesaurus&code=C54100",
                                        },
                                    },
                                    "type": "string",
                                },
                                "submitter_id": {"type": "string"},
                            },
                            "type": "object",
                        },
                        "type": "array",
                    },
                    {
                        "additionalProperties": True,
                        "properties": {
                            "id": {
                                "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
                                "term": {
                                    "description": "A 128-bit identifier. Depending on the mechanism used to generate it, it is either guaranteed to be different from all other UUIDs/GUIDs generated until 3400 AD or extremely likely to be different. Its relatively small size lends itself well to sorting, ordering, and hashing of all sorts, storing in databases, simple allocation, and ease of programming in general.\n",
                                    "termDef": {
                                        "cde_id": "C54100",
                                        "cde_version": None,
                                        "source": "NCIt",
                                        "term": "Universally Unique Identifier",
                                        "term_url": "https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&version=16.02d&ns=NCI_Thesaurus&code=C54100",
                                    },
                                },
                                "type": "string",
                            },
                            "submitter_id": {"type": "string"},
                        },
                        "type": "object",
                    },
                ],
                "description": "The program to which the project belongs.\n",
            }
        # add link name to required properties (and any other Gen3-required project props)
        if "required" in gdm["project"] and "program" not in gdm["project"]["required"]:
            gdm["project"]["required"].append("program")

        # Add required properties to project if missing
        required_props = ["code", "name", "dbgap_accession_number"]
        id_prop = {
            "description": "UUID for the project.",
            "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
            "systemAlias": "node_id",
            "term": {
                "description": "A 128-bit identifier. Depending on the mechanism used to generate it, it is either guaranteed to be different from all other UUIDs/GUIDs generated until 3400 AD or extremely likely to be different. Its relatively small size lends itself well to sorting, ordering, and hashing of all sorts, storing in databases, simple allocation, and ease of programming in general.\n",
                "termDef": {
                    "cde_id": "C54100",
                    "cde_version": None,
                    "source": "NCIt",
                    "term": "Universally Unique Identifier",
                    "term_url": "https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&version=16.02d&ns=NCI_Thesaurus&code=C54100",
                },
            },
            "type": "string",
        }
        name_prop = {
            "description": "Display name/brief description for the project.",
            "type": "string",
        }
        dbgap_accession_number_prop = {
            "description": "The dbgap accession number provided for the project.",
            "type": "string",
        }
        for prop in required_props:
            if prop not in gdm["project"]["properties"]:
                if prop == "id":
                    gdm["project"]["properties"][prop] = id_prop
                elif prop == "name":
                    gdm["project"]["properties"][prop] = name_prop
                elif prop == "dbgap_accession_number":
                    gdm["project"]["properties"][prop] = dbgap_accession_number_prop
            if prop not in gdm["project"]["required"]:
                gdm["project"]["required"].append(prop)
    return gdm


# gdm = fix_project(gdm)


## Add required Gen3 properties (submitter_id, id, code, name etc.) to the Gen3-style data model nodes
def add_gen3_required_properties(gdm):
    """Adds the required Gen3 properties to the project node in the Gen3-style data model.
    Ensures that the project node has: code, name, dbgap_accession_number, program.
    Ensures other nodes have the required/system properties: id, submitter_id, created_datetime, updated_datetime, state, project_id,
    Adds "submitter_id" and "type" to "required"
    if "file_name", "object_id", "data_format", "data_category", "data_type", "md5sum" etc. are present, they need to add file_properties to properties/required as well.
    """
    # Define the required props that could be missing from simplified data models

    required_props = {
        "created_datetime": {
            "oneOf": [{"format": "date-time", "type": "string"}, {"type": "null"}],
            "term": {
                "description": "A combination of date and time of day in the form [-]CCYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm]\n"
            },
        },
        "id": {
            "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
            "systemAlias": "node_id",
            "term": {
                "description": "A 128-bit identifier. Depending on the mechanism used to generate it, it is either guaranteed to be different from all other UUIDs/GUIDs generated until 3400 AD or extremely likely to be different. Its relatively small size lends itself well to sorting, ordering, and hashing of all sorts, storing in databases, simple allocation, and ease of programming in general.\n",
                "termDef": {
                    "cde_id": "C54100",
                    "cde_version": None,
                    "source": "NCIt",
                    "term": "Universally Unique Identifier",
                    "term_url": "https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&version=16.02d&ns=NCI_Thesaurus&code=C54100",
                },
            },
            "type": "string",
        },
        "project_id": {
            "term": {
                "description": "Unique ID for any specific defined piece of work that is undertaken or attempted to meet a single requirement.\n"
            },
            "type": "string",
        },
        "state": {
            "default": "validated",
            "downloadable": [
                "uploaded",
                "md5summed",
                "validating",
                "validated",
                "error",
                "invalid",
                "released",
            ],
            "oneOf": [
                {
                    "enum": [
                        "uploading",
                        "uploaded",
                        "md5summing",
                        "md5summed",
                        "validating",
                        "error",
                        "invalid",
                        "suppressed",
                        "redacted",
                        "live",
                    ]
                },
                {"enum": ["validated", "submitted", "released"]},
            ],
            "public": ["live"],
            "term": {"description": "The current state of the object.\n"},
        },
        "submitter_id": {
            "description": "A human-readable, unique identifier for a record in the metadata database. It can be used in place of the UUID for identifying or recalling a record (e.g., in data queries or uploads/exports).",
            "type": "string",
        },
        "type": {
            "description": 'The node_id of the node in the data model; the name of the node used in queries and API requests (e.g., "aligned_reads_file" for the "Aligned Reads File" node).',
            "type": "string",
        },
        "updated_datetime": {
            "oneOf": [{"format": "date-time", "type": "string"}, {"type": "null"}],
            "term": {
                "description": "A combination of date and time of day in the form [-]CCYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm]\n"
            },
        },
    }
    for node in [
        n
        for n in gdm
        if n
        not in [
            "_definitions",
            "_settings",
            "_terms",
            "program",
            "project",
            "core_metadata_collection",
            "data_release",
            "metaschema",
            "root",
        ]
    ]:
        for req in list(required_props.keys()):
            if req not in gdm[node]["properties"]:
                gdm[node]["properties"][req] = required_props[req]
        # add submitter_id and type to required_properties
        if "submitter_id" not in gdm[node]["required"]:
            gdm[node]["required"].append("submitter_id")
        if "type" not in gdm[node]["required"]:
            gdm[node]["required"].append("type")
        if f"{node}.id" in gdm[node]["required"]:
            gdm[node]["required"].remove(f"{node}.id")
        # replace the link names with .id with the target_node name
        link_targets = [link["target_type"] for link in gdm[node]["links"]]
        for link in link_targets:
            if f"{link}.id" in gdm[node]["required"]:
                gdm[node]["required"].remove(f"{link}.id")
            if link not in gdm[node]["required"]:
                gdm[node]["required"].append(link)
    return gdm


# gdm = add_gen3_required_properties(gdm)  # add required Gen3 properties to the project node


def add_yaml_suffix_to_nodes(schema):
    """To ensure that the schema is compatible with Gen3's PFB format:
    Adds a .yaml suffix to all nodes in the schema that do not already have it.
    """
    schema = {
        f"{node}.yaml": schema[node] for node in schema if not node.endswith(".yaml")
    }
    return schema


# gdm = add_yaml_suffix_to_nodes(gdm)  # ensure all nodes have .yaml suffix


def get_md5sum(filename):
    """Return the MD5 hash of a file."""
    hash_md5 = hashlib.md5()
    with open(filename, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()


# md5sum(tsv_path)