Repository: psrenergy/PSRClassesInterface.jl
Pull request #239: [WIP] Add docstring generator
Author: login: pedroripper
Created: 2025-04-15T16:30:20Z | Merged: 2025-04-17T17:25:21Z

Objective: Generate automatic docstrings for complex collections

Current state:

- load a study using PSRDatabaseSQLite
- Add the model database folder that contains a toml directory containing metadata for each collection

```
julia> PSRDatabaseSQLite.parameters_docstring(study.collections_map["HydroUnit"]; model_database_folder= raw"D:\git\IARA.jl\database")
Required arguments:
- `label::String`: Label of the hydro unit.
- `waveguide_volume::Vector{Float64}`: Tiebreaker curve values of the virtual reservoir `[GWh]`.
- `parameters::DataFrames.DataFrame: A dataframe containing time series attributes (described below).`


Optional arguments:
- `initial_volume::Float64`: Initial volume of the hydro unit.
- `initial_volume_type::Int64`: Initial volume type of the hydro unit (default: `2`).
- `has_commitment::Int64`: Has commitment of the hydro unit (default: `0`).
- `operation_type::Int64`: Operation type of the hydro unit (default: `0`).
- `hydrounit_spill_to::Int64`: Spill to of the hydro unit.
- `hydrounit_turbine_to::Int64`: Turbine to of the hydro unit.
- `gaugingstation_id::Int64`: Gauging station of the hydro unit.
- `biddinggroup_id::Int64`: Bidding group of the hydro unit.
- `bus_id::Int64`: Bus of the hydro unit.


---

** Time Series Attributes **

Group `parameters`:
- `date_time::Vector{DateTime}`: date and time of the time series.
- `existing::Vector{Int64}`: Existing of the hydro unit.
- `production_factor::Vector{Float64}`: Production factor of the hydro unit `[MW/m³/s]`.
- `min_generation::Vector{Float64}`: Min generation of the hydro unit `[MW]`.
- `max_generation::Vector{Float64}`: Max generation of the hydro unit `[MW]`.
- `max_turbining::Vector{Float64}`: Max turbining of the hydro unit `[m³/s]`.
- `min_volume::Vector{Float64}`: Min volume of the hydro unit `[hm³]`.
- `max_volume::Vector{Float64}`: Max volume of the hydro unit `[hm³]`.
- `min_outflow::Vector{Float64}`: Min outflow of the hydro unit `[m³/s]`.
- `om_cost::Vector{Float64}`: O&M cost of the hydro unit `[$/MWh]`.

```


Diff:
diff --git a/Project.toml b/Project.toml
index 7dcf25d6..99f06052 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,6 +1,6 @@
 name = "PSRClassesInterface"
 uuid = "1eab49e5-27d8-4905-b9f6-327b6ea666c4"
-version = "0.17.7"
+version = "0.17.8"
 
 [deps]
 DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
@@ -11,6 +11,7 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
@@ -26,6 +27,7 @@ SQLite = "1.6.0"
 Tables = "1.10"
 Test = "1.6"
 julia = "1.6"
+TOML = "1"
 
 [extras]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
diff --git a/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl b/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl
index 7bf589bf..885897d6 100644
--- a/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl
+++ b/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl
@@ -29,5 +29,6 @@ include("delete.jl")
 include("validate.jl")
 include("migrations.jl")
 include("psri_study_interface.jl")
+include("docstrings.jl")
 
 end # module DatabaseSQLite
diff --git a/src/PSRDatabaseSQLite/docstrings.jl b/src/PSRDatabaseSQLite/docstrings.jl
new file mode 100644
index 00000000..5d1aa436
--- /dev/null
+++ b/src/PSRDatabaseSQLite/docstrings.jl
@@ -0,0 +1,281 @@
+using TOML
+
+struct GroupMap
+    group_id::String
+    parameters::Vector{TimeSeries{<:Number}}
+    dimensions::Vector{String}
+end
+
+function _snake_case(string::String)
+    string = replace(string, r"[\-\.\s]" => "_")
+    words = lowercase.(split(string, r"(?=[A-Z])"))
+    return join([i == 1 ? word : "_$word" for (i, word) in enumerate(words)])
+end
+
+function _get_parameter_metadata(parameter::Attribute, toml_map::Dict{Any, Any}, enum_map::Dict{String, Any})
+    metadata = ""
+    if haskey(toml_map, parameter.id)
+        if haskey(toml_map[parameter.id], "tooltip")
+            tooltip = if typeof(toml_map[parameter.id]["tooltip"]) == String
+                toml_map[parameter.id]["tooltip"]
+            else
+                toml_map[parameter.id]["tooltip"]["en"]
+            end
+            if !ismissing(tooltip)
+                metadata *= " $tooltip"
+            end
+        end
+        if haskey(toml_map[parameter.id], "unit")
+            unit = if typeof(toml_map[parameter.id]["unit"]) == String
+                toml_map[parameter.id]["unit"]
+            else
+                toml_map[parameter.id]["unit"]["en"]
+            end
+            if !ismissing(unit)
+                metadata *= " `[$unit]`"
+            end
+        end
+        if haskey(toml_map[parameter.id], "enum")
+            metadata *= "\n"
+            for enum_value in enum_map[toml_map[parameter.id]["enum"]]
+                value = if typeof(enum_value["label"]) == String
+                    enum_value["label"]
+                else
+                    enum_value["label"]["en"]
+                end
+                metadata *= "   - `$(enum_value["id"])` [$(value)]"
+                if !ismissing(parameter.default_value) && (parameter.default_value == enum_value["id"])
+                    metadata *= " <default> \n"
+                else
+                    metadata *= "\n"
+                end
+            end
+        elseif !ismissing(parameter.default_value)
+            metadata *= " <default `$(parameter.default_value)`> \n"
+        else
+            metadata *= "\n"
+        end
+    end
+    if isempty(metadata)
+        return ""
+    else
+        return ":$metadata"
+    end
+end
+
+function _get_collection_toml(collection::Collection, toml_path::String)
+    toml_map = Dict()
+
+    toml_reader = TOML.parsefile(toml_path)
+
+    for attribute in toml_reader["attribute"]
+        toml_map[attribute["id"]] = attribute
+    end
+
+    if haskey(toml_reader, "attribute_group")
+        for attribute_group in toml_reader["attribute_group"]
+            toml_map[attribute_group["id"]] = attribute_group
+        end
+    end
+
+    return toml_map
+end
+
+function _get_enum_toml(collection::Collection, toml_path::String)
+    if isfile(toml_path)
+        toml_reader = TOML.parsefile(toml_path)
+        return toml_reader
+    end
+    return Dict()
+end
+
+function _get_time_series_groups_map(
+    parameters::OrderedDict{String, <:TimeSeries},
+)
+    groups_map = Dict{String, GroupMap}()
+    for (key, parameter) in parameters
+        if haskey(groups_map, parameter.group_id)
+            push!(groups_map[parameter.group_id].parameters, parameter)
+        else
+            groups_map[parameter.group_id] = GroupMap(
+                parameter.group_id,
+                [parameter],
+                parameter.dimension_names,
+            )
+        end
+    end
+    return groups_map
+end
+
+function _generate_docstrings(
+    time_series_groups::Dict{String, GroupMap},
+    toml_map::Dict{Any, Any},
+    enum_map::Dict{String, Any},
+)
+    arguments = ""
+
+    for (key, group) in time_series_groups
+        entry = "Group `$(key)`:\n"
+        for dimension in group.dimensions
+            if dimension == "date_time"
+                entry *= "- `date_time::Vector{DateTime}`: date and time of the time series \n"
+            else
+                entry *= "- `$(dimension)::Vector{Int64}`: dimension of the time series \n"
+            end
+        end
+        for parameter in group.parameters
+            entry *= "- `$(parameter.id)::Vector{$(parameter.type)}`"
+            entry *= _get_parameter_metadata(parameter, toml_map, enum_map)
+            if !ismissing(parameter.default_value)
+                entry *= " (default: `$(parameter.default_value)`)"
+            end
+        end
+        arguments *= entry
+    end
+    if length(arguments) > 0
+        divider = """
+        ---
+
+        **Time Series Attributes**\n
+        """
+        arguments = divider * arguments
+    end
+
+    return arguments
+end
+
+function _generate_docstrings(
+    parameters::OrderedDict{String, <:ScalarAttribute},
+    toml_map::Dict{Any, Any},
+    enum_map::Dict{String, Any};
+    ignore_id::Bool = true,
+)
+    required_arguments = ""
+    optional_arguments = ""
+
+    for (key, parameter) in parameters
+        if ignore_id && parameter.id == "id"
+            continue
+        end
+        if parameter.not_null
+            entry = "- `$(parameter.id)::$(parameter.type)`"
+            entry *= _get_parameter_metadata(parameter, toml_map, enum_map)
+            required_arguments *= entry * " \n"
+        else
+            entry = "- `$(parameter.id)::$(parameter.type)`"
+            entry *= _get_parameter_metadata(parameter, toml_map, enum_map)
+            optional_arguments *= entry * " \n"
+        end
+    end
+    return required_arguments, optional_arguments
+end
+
+function _generate_docstrings(
+    parameters::OrderedDict{String, <:VectorAttribute},
+    toml_map::Dict{Any, Any},
+    enum_map::Dict{String, Any};
+    ignore_id::Bool = true,
+)
+    required_arguments = ""
+    optional_arguments = ""
+
+    for (key, parameter) in parameters
+        if ignore_id && parameter.id == "id"
+            continue
+        end
+        if parameter.not_null
+            entry = "- `$(parameter.id)::Vector{$(parameter.type)}`"
+            entry *= _get_parameter_metadata(parameter, toml_map, enum_map)
+            required_arguments *= entry * " \n"
+        else
+            entry = "- `$(parameter.id)::Vector{$(parameter.type)}`"
+            entry *= _get_parameter_metadata(parameter, toml_map, enum_map)
+            optional_arguments *= entry * " \n"
+        end
+    end
+    return required_arguments, optional_arguments
+end
+
+function collection_docstring(
+    model_folder::String,
+    collection::String;
+    ignore_id::Bool = true,
+)
+    docstring = ""
+
+    mktempdir() do temp_folder
+        study = create_empty_db_from_migrations(
+            joinpath(temp_folder, "$(collection)_study.db"),
+            joinpath(model_folder, "migrations"),
+        )
+
+        collection = study.collections_map[collection]
+
+        toml_path = joinpath(model_folder, "ui", _snake_case(collection.id) * ".toml")
+        attribute_toml_map = _get_collection_toml(collection, toml_path)
+
+        enum_toml_map = _get_enum_toml(collection, joinpath(model_folder, "ui", "enum.toml"))
+        required_arguments = ""
+        optional_arguments = ""
+        time_series_arguments = ""
+
+        scalar_required, scalar_optional = _generate_docstrings(
+            collection.scalar_parameters,
+            attribute_toml_map,
+            enum_toml_map;
+            ignore_id = ignore_id,
+        )
+        required_arguments *= scalar_required
+        optional_arguments *= scalar_optional
+
+        scalar_relation_required, scalar_relation_optional = _generate_docstrings(
+            collection.scalar_relations,
+            attribute_toml_map,
+            enum_toml_map;
+            ignore_id = ignore_id,
+        )
+        required_arguments *= scalar_relation_required
+        optional_arguments *= scalar_relation_optional
+
+        vector_required, vector_optional = _generate_docstrings(
+            collection.vector_parameters,
+            attribute_toml_map,
+            enum_toml_map;
+            ignore_id = ignore_id,
+        )
+        required_arguments *= vector_required
+        optional_arguments *= vector_optional
+
+        if length(required_arguments) > 0
+            required_arguments = "Required arguments:\n" * required_arguments
+        end
+        if length(optional_arguments) > 0
+            optional_arguments = "Optional arguments:\n" * optional_arguments
+        end
+
+        time_series_groups = _get_time_series_groups_map(
+            collection.time_series,
+        )
+
+        for group in keys(time_series_groups)
+            required_arguments *= "- `$(group)::DataFrames.DataFrame: A dataframe containing time series attributes (described below).`\n"
+        end
+
+        time_series_arguments = _generate_docstrings(
+            time_series_groups,
+            attribute_toml_map,
+            enum_toml_map,
+        )
+
+        docstring *= """
+        $required_arguments
+        $optional_arguments
+        $time_series_arguments
+        """
+
+        close!(study)
+        return rm(study.database_path; force = true)
+    end
+
+    return docstring
+end


Reviews:
user: login: guilhermebodin
body: São detalhes, depois que ajustar pode dar merge
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T03:32:37Z
user: login: guilhermebodin
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-17T17:23:37Z

Review comments:
path: script.jl
user: login: guilhermebodin
body: Tem que tirar esse script do commit
created_at: 2025-04-17T03:29:57Z
author_association: MEMBER
diff_hunk: @@ -26,6 +27,7 @@ SQLite = "1.6.0"
 Tables = "1.10"
 Test = "1.6"
 julia = "1.6"
+TOML = "1.0.0"
path: Project.toml
user: login: guilhermebodin
body: Poderia ser TOML = "1"
created_at: 2025-04-17T03:30:09Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,274 @@
+using TOML
+
+function _snake_case(string)
+    string = replace(string, r"[\-\.\s]" => "_")
+    words = lowercase.(split(string, r"(?=[A-Z])"))
+    return join([i == 1 ? word : "_$word" for (i, word) in enumerate(words)])
+end
+
+function _get_parameter_metadata(parameter::PSRDatabaseSQLite.Attribute, toml_map::Dict{Any, Any}, enum_map)
+    metadata = ""
+    if haskey(toml_map, parameter.id)
+        if haskey(toml_map[parameter.id], "tooltip")
+            tooltip = if typeof(toml_map[parameter.id]["tooltip"]) == String
+                toml_map[parameter.id]["tooltip"]
+            else
+                toml_map[parameter.id]["tooltip"]["en"]
+            end
+            if !ismissing(tooltip)
+                metadata *= " $tooltip"
+            end
+        end
+        if haskey(toml_map[parameter.id], "unit")
+            unit = if typeof(toml_map[parameter.id]["unit"]) == String
+                toml_map[parameter.id]["unit"]
+            else
+                toml_map[parameter.id]["unit"]["en"]
+            end
+            if !ismissing(unit)
+                metadata *= " `[$unit]`"
+            end
+        end
+        if haskey(toml_map[parameter.id], "enum")
+            metadata *= "\n"
+            for enum_value in enum_map[toml_map[parameter.id]["enum"]]
+                value = if typeof(enum_value["label"]) == String
+                    enum_value["label"]
+                else
+                    enum_value["label"]["en"]
+                end
+                metadata *= "   - `$(enum_value["id"])` [$(value)]"
+                if !ismissing(parameter.default_value) && (parameter.default_value == enum_value["id"])
+                    metadata *= " <default> \n"
+                else
+                    metadata *= "\n"
+                end
+            end
+        elseif !ismissing(parameter.default_value)
+            metadata *= " <default `$(parameter.default_value)`> \n"
+        else
+            metadata *= "\n"
+        end
+    end
+    if isempty(metadata)
+        return ""
+    else
+        return ":$metadata"
+    end
+end
+
+function _get_collection_toml(collection::PSRDatabaseSQLite.Collection, toml_path::String)
+    toml_map = Dict()
+
+    toml_reader = TOML.parsefile(toml_path)
+
+    for attribute in toml_reader["attribute"]
+        toml_map[attribute["id"]] = attribute
+    end
+
+    if haskey(toml_reader, "attribute_group")
+        for attribute_group in toml_reader["attribute_group"]
+            toml_map[attribute_group["id"]] = attribute_group
+        end
+    end
+
+    return toml_map
+end
+
+function _get_enum_toml(collection::PSRDatabaseSQLite.Collection, toml_path::String)
+    if isfile(toml_path)
+        toml_reader = TOML.parsefile(toml_path)
+        return toml_reader
+    end
+    return Dict()
+end
+
+function _generate_scalar_docstrings(
+    parameters::OrderedCollections.OrderedDict{String, <:PSRClassesInterface.PSRDatabaseSQLite.ScalarAttribute},
path: src/PSRDatabaseSQLite/docstrings.jl
user: login: guilhermebodin
body: Acho que não precisa especificar tanto esses tipos
created_at: 2025-04-17T03:31:16Z
author_association: MEMBER

PR comments:
user: login: pedroripper
created_at: 2025-04-15T20:36:57Z
body: Update: Now adding enum values

```

julia> print(PSRDatabaseSQLite.collection_docstring(study.collections_map["HydroUnit"]; model_database_folder= raw"D:\git\IARA.jl\database"))
Required arguments:
- `label::String`: Label of the hydro unit
- `waveguide_volume::Vector{Float64}`: Tiebreaker curve values of the virtual reservoir `[GWh]`

- `parameters::DataFrames.DataFrame: A dataframe containing time series attributes (described below).`


Optional arguments:
- `initial_volume::Float64`: Initial volume of the hydro unit
- `initial_volume_type::Int64`: Initial volume type of the hydro unit
   - `0` [Per Unit]
   - `2` [Volume] (default)
- `has_commitment::Int64`: Has commitment of the hydro unit
   - `0` [No Commitment] (default)
   - `1` [Has Commitment]
- `operation_type::Int64`: Operation type of the hydro unit
   - `0` [Reservoir] (default)
   - `1` [Run of River]
- `hydrounit_spill_to::Int64`: Spill to of the hydro unit
- `hydrounit_turbine_to::Int64`: Turbine to of the hydro unit
- `gaugingstation_id::Int64`: Gauging station of the hydro unit
- `biddinggroup_id::Int64`: Bidding group of the hydro unit
- `bus_id::Int64`: Bus of the hydro unit


---

** Time Series Attributes **

Group `parameters`:
- `date_time::Vector{DateTime}`: date and time of the time series
- `existing::Vector{Int64}`: Existing of the hydro unit
   - `0` [Does Not Exist]
   - `1` [Exists]
- `production_factor::Vector{Float64}`: Production factor of the hydro unit `[MW/m³/s]`
- `min_generation::Vector{Float64}`: Min generation of the hydro unit `[MW]`
- `max_generation::Vector{Float64}`: Max generation of the hydro unit `[MW]`
- `max_turbining::Vector{Float64}`: Max turbining of the hydro unit `[m³/s]`
- `min_volume::Vector{Float64}`: Min volume of the hydro unit `[hm³]`
- `max_volume::Vector{Float64}`: Max volume of the hydro unit `[hm³]`
- `min_outflow::Vector{Float64}`: Min outflow of the hydro unit `[m³/s]`
- `om_cost::Vector{Float64}`: O&M cost of the hydro unit `[$/MWh]`

```
author_association: MEMBER
user: login: guilhermebodin
created_at: 2025-04-17T03:34:23Z
body: @pedroripper tem que fazer bump da versão e publicar uma nova
author_association: MEMBER