utils/constants.py

from pathlib import Path

NORM_BASE_SUBMISSION = "114be1f0-5a41-43a5-b4e6-7fb683bc01ec"
DIGITS_FOR_VALUES = 3
DIGITS_FOR_ERRORS = 6

DIMENSIONS = {
            "Segmentation": ['chesapeake', 'sa_crop_type', 'pv4ger_seg', 'cashew', 'neontree', 'nz_cattle'],
            "Classification": ['pv4ger', 'so2sat', 'brick_kiln', 'big_earth_net', 'eurosat', 'forestnet'],
            "High Resolution": ['chesapeake', 'pv4ger_seg', 'pv4ger', 'nz_cattle', 'neontree' ],
            "Medium Resolution": ['brick_kiln', 'big_earth_net', 'eurosat', 'so2sat','sa_crop_type', 'cashew', 'forestnet'],
            "RGB only": ['pv4ger_seg','chesapeake', 'nz_cattle', 'neontree', 'pv4ger'],
            "Multi-Spectral": [ 'sa_crop_type', 'cashew','so2sat', 'brick_kiln', 'big_earth_net', 'eurosat', 'forestnet']
            }

DIMENSION_INFO = {
            "Segmentation": "datasets for pixel-wise semantic segmentation",
            "Classification": "single-label and multi-label classification datasets",
            "High Resolution": "datasets with resolution <= 1 metre",
            "Medium Resolution": "datasets with 10 metres <= resolution <= 15 metres",
            "RGB only": "datasets using only Red, Green, and Blue bands",
            "Multi-Spectral": " datasets with wavelengths beyond the visible spectrum"
            }


DATASETS = [
            'big_earth_net', 'so2sat', 'brick_kiln', 'forestnet', 'eurosat', 'pv4ger',  
            'pv4ger_seg', 'chesapeake', 'cashew', 'sa_crop_type', 'nz_cattle', 'neontree'
            ]


DATASET_INFO = {
                "Dataset": [item.replace("_", " ").title() for item in DATASETS],
                "Description": [
                    "land cover dataset with multi-label classification in 10 European countries",
                    "local climate zone classification dataset with global coverage",
                    "dataset of brick kilns detected by satellite in Bangladesh",
                    "classification dataset of known forest loss events paired with driver annotations from expert interpreters",
                    "land use/land cover classification dataset",
                    "classification dataset for photo-voltaic systems in Germany",
                    "segmentation dataset for photo-voltaic systems in Germany",
                    "high resolution land cover (segmentation) dataset of Chesapeake Bay, US",
                    "segmentation dataset for cashew plantations in Benin",
                    "segmentation dataset with ground reference crop type labels and multispectral imagery from Germany and South Africa",
                    "segmentation dataset for detecting cows from high resolution aerial images in New Zealand",
                    "segmentation dataset for canopy crown detection and delineation in the US"
                ],
                "Image Size": ["120 x 120", "32 x 32", "64 x 64", "332 x 332", "64 x 64", "320 x 320", 
                               "320 x 320", "256 x 256", "256 x 256","256 x 256", "500 x 500", "400 x 400"],
                "# Classes": [43, 17, 2, 12, 10, 2, 2, 7, 7, 10, 2, 2],
                "Train": [20000, 19992, 15063, 6464, 2000, 11814, 3000, 3000, 1350, 3000, 524, 270],
                "Val": [1000, 986, 999, 989, 1000, 999, 403, 1000, 400, 1000, 66, 94],
                "Test": [1000, 986, 999, 993, 1000, 999, 403, 1000, 50, 1000, 65, 93],
                "# Bands": [12, 18, 13, 6, 13, 3, 3, 4, 13, 13, 3, 5],
                "RGB Res": [10, 10, 10, 15, 10, 0.1, 0.1, 1.0, 10, 10, 0.1, 0.1],
                "Sensors": ["Sentinel-2","Sentinel-2", "Sentinel-2", "Landsat-8", "Sentinel-2", "RGB", 
                            "RGB", "RGBN", "Sentinel-2", "Sentinel-2", "RGB", "RGB"],
                "Metric": ["Multilabel_F1_Score", "Overall_Accuracy", "Overall_Accuracy", "Overall_Accuracy", "Overall_Accuracy", "Overall_Accuracy",
                           "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index", "Multiclass_Jaccard_Index"],
                "Citation": ["Sumbul et al. [2021]",  "Zhu et al [2019]", "Lee et al [2021]", "Irvin et al [2020]", "Helber et al [2019]", "Mayer et al [2022]",
                             "Mayer et al [2022]", "Robinson et al [2019]", "Z. et al [2021]", "link", "Abuaiadah and Switzer [2022]", "Weinstein et al. [2021]"
                             ],
                "Hyperlinks": [
                                "https://arxiv.org/abs/2105.07921",
                                "https://arxiv.org/abs/1912.12171",
                                "https://www.pnas.org/content/118/17/e2018863118",
                                "https://arxiv.org/abs/2011.05479",
                                "https://arxiv.org/abs/1709.00029",
                                "https://www.sciencedirect.com/science/article/pii/S0306261921016937",
                                "https://www.sciencedirect.com/science/article/pii/S0306261921016937",
                                "https://ieeexplore.ieee.org/document/8953207",
                                "https://arxiv.org/abs/2301.00363",
                                "https://source.coop/esa/fusion-competition",
                                "https://doi.org/10.5281/zenodo.5908869",
                                "https://doi.org/10.1371/journal.pcbi.1009180"
                                ],
                "License": ["CDLA-P-1.0", "CC-BY-4.0", "CC-BY-SA 4.0", "CC-BY-4.0", "MIT", "MIT", "MIT", "CDLA-P-1.0", "CC-BY-4.0", "CC-BY-4.0", "CC-BY-4.0", "CC0 1.0"],
                "Dimensions": [", ".join([dim for dim, data_list in DIMENSIONS.items() if dataset in data_list]) for dataset in DATASETS]
                }


COLUMN_ORDER = {
                "raw": {
                    "overall_table": ['Overall Mean'] + list(DIMENSIONS.keys()),
                    "dataset_tables": ["Mean"]   ,
                    "dimension_tables": []
                },
                "normalized": {
                    "overall_table": ['Overall IQM'] + list(DIMENSIONS.keys()),
                    "dataset_tables": ["IQM"]   ,
                    "dimension_tables": []
                },
                "all_tables": ['Model', '# params', 'submission', 'Config Settings'],
                }

root = Path(__file__).parent.resolve()
root = "/".join(str(root).split("/")[:-1])
RESULTS_DIR = f"{root}/results"
MODEL_INFO_FILE = f"{root}/utils/model_info.json"
NORMALIZER_DIR = f"{root}/utils/normalizer"


#for validation of new submissions
NEW_SUBMISSION_FOLDER = f"{root}/new_submission"
CSV_FILE = "results_and_parameters.csv"
JSON_FILE = "additional_info.json"
NEW_SUBMISSION_COLUMN_INFO = {
                                "string_cols": ['dataset', 'Metric', 'experiment_name', 'partition name', 'backbone', 'decoder','batch_size_selection'],
                                "integer_cols": ['early_stop_patience', 'n_trials', 'Seed', 'data_percentages', 'batch_size'],
                                "float_cols": ['weight_decay', 'lr', 'test metric', ]
                            }
NEW_SUBMISSION_COLUMN_NAMES = []
for key, value in NEW_SUBMISSION_COLUMN_INFO.items():
    NEW_SUBMISSION_COLUMN_NAMES.extend(value)

                                
JSON_FORMAT = {
                "Paper Link": "N/A",
                "Code Repository Link ": "N/A",
                "License": "N/A",
                "Number of HPO trials": "16",
                "Additional information about submission": "N/A",
                "Comments on new models in submission": "N/A",
                "New model info": 
                    [
                        {
                            "model_display_name": "TBD",
                            "model_size": "TBD", 
                            "unique_backbone_key": "TBD"
                        }
                    ]
                }