如何在python中为我的自定义数据创建流生成器

Question

我做了一个猫/狗的二元分类我用这种方式创建了一个训练数据，我对图像应用了一个平均过滤器。问题是，数据库是相当大的，我得到显示之后，你的笔记本试图分配更多的内存比可用的。我读到python中的生成器占用更少的磁盘内存，可以解决这个问题，但我不知道如何创建一个适合于我刚刚创建作为训练数据的代码的生成器

train_dir = "../input/dog-cat/train"
CATEGORIES = ["dog", "cat"]

training_data = []
def create_training_data():
for category in CATEGORIES:  
path = os.path.join(train_dir,category)  
class_num = CATEGORIES.index(category)  
for img in tqdm(os.listdir(path)):  
try:
img_train = cv2.imread(os.path.join(path,img))
img_mean = cv2.blur(reduced_img_train,(9,9))
training_data.append([img_mean, class_num])  
except Exception as e:
pass
create_training_data()
import random
random.shuffle(training_data)
x_train=[]
y_train=[]
for features,label in training_data:
x_train.append(features)
y_train.append(label)

Answer 1

的要求，你想使用ImageDataGenerator()与模糊函数，检查CV2 CV2.blur()。您可以通过提供的自定义函数"preprocessing_function = custom_image_preprocess";参数

示例:当您可以使用自定义函数或相同的图像通道顺序(在跆拳道颜色游戏中用于重建数据的一种隐藏技术)时，使用标准差的CV2。

import tensorflow as tf
import matplotlib.pyplot as plt
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
None
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
config = tf.config.experimental.set_memory_growth(physical_devices[0], True)
print(physical_devices)
print(config)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Variables
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
BATCH_SIZE = 1
IMG_HEIGHT = 32
IMG_WIDTH = 32
IMG_CHANNELS=3
seed=42
directory = "F:\datasets\downloads\example\image\"
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Definition / Class
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
def custom_image_preprocess( image ):
image = tf.keras.preprocessing.image.array_to_img(
image,
data_format=None,
scale=True
)
img_array = tf.keras.preprocessing.image.img_to_array( image )
img_1 = tf.keras.utils.array_to_img(img_array)

temp = tf.concat([ tf.constant( img_array[:,:,0], shape=(img_array.shape[0], img_array.shape[1], 1) ), tf.constant( 150 - img_array[:,:,1], shape=(img_array.shape[0], img_array.shape[1], 1) ) ], axis=2)
image = tf.concat([ tf.constant( temp[:,:,:], shape=(img_array.shape[0], img_array.shape[1], 2) ), tf.constant( 0.25 * img_array[:,:,2], shape=(img_array.shape[0], img_array.shape[1], 1) ) ], axis=2)
return image

def train_image_gen():
n_zoom_range = tf.where( tf.math.greater_equal( tf.constant( ( 1.0 * IMG_WIDTH ) / ( IMG_HEIGHT * 4 ), dtype=tf.float32 ), tf.constant( 0.25, dtype=tf.float32 ) ), ( 1.0 * IMG_WIDTH ) / ( IMG_HEIGHT * 4 ), 0.25 ).numpy()
n_rotation_range = tf.where( tf.math.greater_equal( tf.constant( ( 1.0 * IMG_WIDTH ) / ( IMG_HEIGHT * 4 ), dtype=tf.float32 ), tf.constant( 0.25, dtype=tf.float32 ) ), ( 1.0 * IMG_WIDTH ) / ( IMG_HEIGHT * 4 ) * 100, 27.25 ).numpy()
n_rescale = tf.where( tf.math.less_equal( tf.constant( 1.0 / ( IMG_WIDTH + IMG_HEIGHT )), tf.constant( 125.0 )), tf.constant( 1.0 / ( IMG_WIDTH + IMG_HEIGHT )).numpy(), 125.0 ).numpy()
train_generator = tf.keras.preprocessing.image.ImageDataGenerator(
# shear_range=0.2,
# zoom_range=float(n_zoom_range),
# horizontal_flip=True,
validation_split=0.2,
# rotation_range=float(n_rotation_range),
# rescale=float(n_rescale),

# rescale=1./255,
# featurewise_center=False,
# samplewise_center=False,
# featurewise_std_normalization=False,
# samplewise_std_normalization=False,
# zca_whitening=False,
# zca_epsilon=1e-06,
# rotation_range=0,
# width_shift_range=0.0,
# height_shift_range=0.0,
# brightness_range=None,
# shear_range=0.0,
# zoom_range=0.0,
# channel_shift_range=0.0,
# fill_mode='nearest',
# cval=0.0,
# horizontal_flip=False,
# vertical_flip=False,
# rescale=None,
preprocessing_function=custom_image_preprocess
# data_format=None,
# validation_split=0.0,
# interpolation_order=1,
# dtype=None
# https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/image/ImageDataGenerator

)

train_image_ds = train_generator.flow_from_directory(
directory,
target_size=(IMG_HEIGHT, IMG_WIDTH),
batch_size=BATCH_SIZE,
class_mode='binary',    # None  # categorical   # binary
subset='training',
color_mode='rgb',       # rgb   # grayscale
seed=seed,
)

return train_image_ds
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Initialize
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model = tf.keras.models.Sequential([
tf.keras.layers.InputLayer(input_shape=( IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS )),
tf.keras.layers.Reshape((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS)),
tf.keras.layers.RandomFlip('horizontal'),
tf.keras.layers.RandomRotation(0.2),
tf.keras.layers.Normalization(mean=3., variance=2.),
tf.keras.layers.Normalization(mean=4., variance=6.),
tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),
tf.keras.layers.Reshape((30, 30, 32)),
tf.keras.layers.MaxPooling2D((2, 2)),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Reshape((128, 225)),
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(96, return_sequences=True, return_state=False)),
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(96)),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(192, activation='relu'),
tf.keras.layers.Dense(10),
])
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Optimizer
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
optimizer = tf.keras.optimizers.Nadam(
learning_rate=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-07,
name='Nadam'
) # 0.00001
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Loss Fn
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""                               
lossfn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Summary
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model.compile(optimizer=optimizer, loss=lossfn, metrics=['accuracy'])
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Training
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
history = model.fit(train_image_gen(), validation_data=train_image_gen(), batch_size=100, epochs=50 )
input( '..;.' )

输出:使用ImageGenerator进行培训，请以监控资源使用为目标。

Found 16 images belonging to 2 classes.
Epoch 1/50
2022-11-26 23:00:06.112861: I tensorflow/stream_executor/cuda/cuda_dnn.cc:368] Loaded cuDNN version 8100
16/16 [==============================] - 9s 146ms/step - loss: 1.1202 - accuracy: 0.4375 - val_loss: 0.7060 - val_accuracy: 0.5000
Epoch 2/50
16/16 [==============================] - 1s 57ms/step - loss: 0.7892 - accuracy: 0.3125 - val_loss: 0.6961 - val_accuracy: 0.5000
Epoch 3/50
3/16 [====>.........................] - ETA: 0s - loss: 0.6903 - accuracy: 0.6667T