Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<create_dataframe>
tez_path = '.. /input/tez-lib/'
effnet_path = '.. /input/efficientnet-pytorch/'
sys.path.append(tez_path)
 sys.path.append(effnet_path)
client = bigquery.Client()
dataset_ref = client.dataset("noaa_gsod", project="bigquery-public-data")
 dataset = client.get_dataset(dataset_ref)
 tables = list(client.list_tables(dataset))
table_ref = dataset_ref.table("stations")
 table = client.get_table(table_ref)
 stations_df = client.list_rows(table ).to_dataframe()
table_ref = dataset_ref.table("gsod2020")
 table = client.get_table(table_ref)
 twenty_twenty_df = client.list_rows(table ).to_dataframe()
stations_df['STN'] = stations_df['usaf'] + '-' + stations_df['wban']
twenty_twenty_df['STN'] = twenty_twenty_df['stn'] + '-' + twenty_twenty_df['wban']
cols_1 = ['STN', 'mo', 'da', 'temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']
cols_2 = ['STN', 'country', 'state', 'call', 'lat', 'lon', 'elev']
weather_df = twenty_twenty_df[cols_1].join(stations_df[cols_2].set_index('STN'), on='STN')
 weather_df.tail(10 )<feature_engineering>
class LeafModel(tez.Model):
def __init__(self, num_classes):
super().__init__()
self.effnet = EfficientNet.from_name("efficientnet-b4")
 self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(1792, num_classes)
 self.step_scheduler_after = "epoch"
def forward(self, image, targets=None):
batch_size, _, _, _ = image.shape
x = self.effnet.extract_features(image)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 outputs = self.out(self.dropout(x))
return outputs, None, None
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in train.index:
df = train.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.Id, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 train['closest_station'] = C
train = train.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 train.sort_values(by=['Id'], inplace=True)
 train.head()<load_from_csv>
test_aug = albumentations.Compose([
albumentations.RandomResizedCrop(256, 256),
albumentations.Transpose(p=0.5),
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
albumentations.RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
albumentations.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0,
p=1.0 
)
 ], p=1.)
test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv" )<feature_engineering>
dfx = pd.read_csv(".. /input/cassava-leaf-disease-classification/sample_submission.csv")
 image_path = ".. /input/cassava-leaf-disease-classification/test_images/"
test_image_paths = [os.path.join(image_path, x)for x in dfx.image_id.values]
test_targets = dfx.label.values
test_dataset = ImageDataset(
 image_paths=test_image_paths,
targets=test_targets,
augmentations=test_aug, 
)
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in test.index:
df = test.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.ForecastId, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 test['closest_station'] = C
test = test.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 test.sort_values(by=['ForecastId'], inplace=True)
 test.head()<data_type_conversions>
train_dfx = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 model = LeafModel(num_classes=train_dfx.label.nunique())
 model.load(".. /input/leafmodel/model.bin")
train["wdsp"] = pd.to_numeric(train["wdsp"])
 test["wdsp"] = pd.to_numeric(test["wdsp"])
 train["fog"] = pd.to_numeric(train["fog"])
 test["fog"] = pd.to_numeric(test["fog"] )<drop_column>
final_preds = None
for j in range(20):
preds = model.predict(test_dataset, batch_size=32, n_jobs=-1, device="cuda")
 temp_preds = None
for p in preds:
if temp_preds is None:
temp_preds = p
else:
temp_preds = np.vstack(( temp_preds, p))
if final_preds is None:
final_preds = temp_preds
else:
final_preds += temp_preds
final_preds /= 20
<data_type_conversions><EOS> 
final_preds = final_preds.argmax(axis=1)
 dfx.label = final_preds
dfx.to_csv("submission.csv", index=False )