Issue 1:
How do I prevent the untar from the above get_file() call from creating duplicate "._*" files?

I get double the images in the dataset when I run the following, and the duplicated images have a prefix of ._

import pathlib

import matplotlib.pyplot as plt

import numpy as np

import PIL

import tensorflow as tf

from tensorflow import keras

from tensorflow.keras import layers

from tensorflow.keras.models import Sequential

custom data attempt

weeds_dataset_url = "https://storage.googleapis.com/aquarium-public/datasets/deepweeds/full_image_set/weed_images.tar"

weeds_data_dir = tf.keras.utils.get_file('weed_images', origin=weeds_dataset_url, untar=True)

weeds_data_dir = pathlib.Path(weeds_data_dir)

snake_weeds = list(weeds_data_dir.glob('Snake weed/*'))

print(snake_weeds[0]) # gives me a file starting with ._

Issue 2:
I'm using a data generator to feed the fit_generator. My generator have as output the tuple (x_val, y_val, val_sample_weights) so showing sample weights. This is like:

import numpy as np
import keras
import librosa
from time import time
import random
from config import *

class DataGenerator(keras.utils.Sequence):

'Generates data for Keras'

def __init__(self, dataframe, batch_size=None, dim=None, labels_dim=None,
n_classes=None, shuffle=True, samples=None, duration=None, sample_weights=None):
'Initialization'
self.dim = dim
self.batch_size = batch_size

self.dataframe = dataframe
self.dataframe = self.dataframe.sample(n=len(self.dataframe))
self.samples = samples
self.on_epoch_end()
self.shuffle = shuffle
self.sample_weights = sample_weights

def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.dataframe) / self.batch_size))

def __getitem__(self, index):
'Generate one batch of data'
random_pd = self.dataframe.iloc[self.batch_size*index : (index+1)*self.batch_size]
# Generate data
X, y = self.__data_generation(random_pd)
return X, y
def __data_generation(self, random_pd):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, 1, self.samples))
y = np.empty((self.batch_size, self.n_classes))
i = 0
while i < self.batch_size:
for index, row in random_pd.iterrows():
# generate
y[i,] = label
X[i,] = ...
i += 1
return X, y, self.sample_weights

so it will return X, y, self.sample_weights.

The problem is that we will get a StopIteration: too many values to unpack as it would expect 2 but I'm giving 3 values - as actually I'm doing.

Traceback (most recent call last):
File "train.py", line 438, in <module>
train()
File "train.py", line 422, in train
callbacks=callbacks
File "/usr/local/lib/python2.7/dist-packages/keras/legacy/interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/keras/models.py", line 1315, in fit_generator
initial_epoch=initial_epoch)
File "/usr/local/lib/python2.7/dist-packages/keras/legacy/interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py", line 2250, in fit_generator
max_queue_size=max_queue_size)
File "/usr/local/lib/python2.7/dist-packages/keras/legacy/interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py", line 2383, in evaluate_generator
generator_output = next(output_generator)
File "/usr/local/lib/python2.7/dist-packages/keras/utils/data_utils.py", line 584, in get
six.raise_from(StopIteration(e), e)
File "/usr/local/lib/python2.7/dist-packages/six.py", line 737, in raise_from
raise value
StopIteration: too many values to unpack

I call the fit_generator as usual passing my training_generator then

history = model.fit_generator(generator=training_generator,
class_weight=class_weights,
verbose=1,
use_multiprocessing=True,
workers=24,
steps_per_epoch=training_steps_per_epoch,
epochs=epochs,
validation_data=validation_generator,
validation_steps = validation_steps_per_epoch,
callbacks=callbacks
)

I do this because when using the fit_generator it is not possibile to pass the sample_weight, since the method signature only supports the class_weight - see here #11800