|
|
import os |
|
|
import pickle |
|
|
|
|
|
from dassl.data.datasets import DATASET_REGISTRY, Datum, DatasetBase |
|
|
from dassl.utils import mkdir_if_missing |
|
|
|
|
|
from .oxford_pets import OxfordPets |
|
|
from .dtd import DescribableTextures as DTD |
|
|
|
|
|
NEW_CNAMES = { |
|
|
"AnnualCrop": "Annual Crop Land", |
|
|
"Forest": "Forest", |
|
|
"HerbaceousVegetation": "Herbaceous Vegetation Land", |
|
|
"Highway": "Highway or Road", |
|
|
"Industrial": "Industrial Buildings", |
|
|
"Pasture": "Pasture Land", |
|
|
"PermanentCrop": "Permanent Crop Land", |
|
|
"Residential": "Residential Buildings", |
|
|
"River": "River", |
|
|
"SeaLake": "Sea or Lake", |
|
|
} |
|
|
|
|
|
|
|
|
@DATASET_REGISTRY.register() |
|
|
class EuroSAT(DatasetBase): |
|
|
|
|
|
dataset_dir = "eurosat" |
|
|
|
|
|
def __init__(self, cfg): |
|
|
root = os.path.abspath(os.path.expanduser(cfg.DATASET.ROOT)) |
|
|
self.dataset_dir = os.path.join(root, self.dataset_dir) |
|
|
self.image_dir = os.path.join(self.dataset_dir, "2750") |
|
|
self.split_path = os.path.join(self.dataset_dir, "split_zhou_EuroSAT.json") |
|
|
self.split_fewshot_dir = os.path.join(self.dataset_dir, "split_fewshot") |
|
|
mkdir_if_missing(self.split_fewshot_dir) |
|
|
|
|
|
if os.path.exists(self.split_path): |
|
|
train, val, test = OxfordPets.read_split(self.split_path, self.image_dir) |
|
|
else: |
|
|
train, val, test = DTD.read_and_split_data(self.image_dir, new_cnames=NEW_CNAMES) |
|
|
OxfordPets.save_split(train, val, test, self.split_path, self.image_dir) |
|
|
|
|
|
num_shots = cfg.DATASET.NUM_SHOTS |
|
|
if num_shots >= 1: |
|
|
seed = cfg.SEED |
|
|
preprocessed = os.path.join(self.split_fewshot_dir, f"shot_{num_shots}-seed_{seed}.pkl") |
|
|
|
|
|
if os.path.exists(preprocessed): |
|
|
print(f"Loading preprocessed few-shot data from {preprocessed}") |
|
|
with open(preprocessed, "rb") as file: |
|
|
data = pickle.load(file) |
|
|
train, val = data["train"], data["val"] |
|
|
else: |
|
|
train = self.generate_fewshot_dataset(train, num_shots=num_shots) |
|
|
val = self.generate_fewshot_dataset(val, num_shots=min(num_shots, 4)) |
|
|
data = {"train": train, "val": val} |
|
|
print(f"Saving preprocessed few-shot data to {preprocessed}") |
|
|
with open(preprocessed, "wb") as file: |
|
|
pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL) |
|
|
|
|
|
subsample = cfg.DATASET.SUBSAMPLE_CLASSES |
|
|
|
|
|
if cfg.TRAINER.NAME == "PromptKD": |
|
|
if cfg.TRAINER.MODAL == "base2novel": |
|
|
train_x, _, _ = OxfordPets.subsample_classes(train, val, test, subsample='all') |
|
|
_, _, test_base = OxfordPets.subsample_classes(train, val, test, subsample='base') |
|
|
_, _, test_novel = OxfordPets.subsample_classes(train, val, test, subsample='new') |
|
|
super().__init__(train_x=train_x, val=test_base, test=test_novel) |
|
|
elif cfg.TRAINER.MODAL == "cross": |
|
|
train, _, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) |
|
|
super().__init__(train_x=train, val=test, test=test) |
|
|
else: |
|
|
train, _, test = OxfordPets.subsample_classes(train, val, test, subsample=subsample) |
|
|
super().__init__(train_x=train, val=test, test=test) |
|
|
|
|
|
|
|
|
self.all_classnames = OxfordPets.get_all_classnames(train, val, test) |
|
|
|
|
|
def update_classname(self, dataset_old): |
|
|
dataset_new = [] |
|
|
for item_old in dataset_old: |
|
|
cname_old = item_old.classname |
|
|
cname_new = NEW_CLASSNAMES[cname_old] |
|
|
item_new = Datum(impath=item_old.impath, label=item_old.label, classname=cname_new) |
|
|
dataset_new.append(item_new) |
|
|
return dataset_new |
|
|
|
