深度学习-加载训练数据方式

方式一 : 加载图片到内存

1.优点 :
不用对数据进行分类
2.缺点 :
对内存要求高,如果数据量过大,内存容量不够,容易造成电脑系统崩溃
3.适用文件存储方式 :
图片没有根据标签分文件夹

4.代码实现

1. 定义文件夹路径

TRAIN_DATA_PATH = 'E:/mldata/dogvscat/data/train/'
TEST_DATA_PATH = 'E:/mldata/dogvscat/data//test1/'

2. 读取图片制作数据集

imgs_per_cat = 1000
image_size = (200, 200)
labels = []
train_images = []
for item in os.listdir(TRAIN_DATA_PATH):
    print("train----" + str(item))
    img = image.load_img(os.path.join(TRAIN_DATA_PATH, str(item)), target_size=image_size)
    img = image.img_to_array(img)
    img = img / 255.
    train_images.append(img)
    labels.append(item.split('.')[0])
X = np.array(train_images)

y = []
for item in labels:
    print("labels------" + str(item))
    if item == 'cat':
        y.append(0)
    else:
        y.append(1)
new_labels = pd.get_dummies(labels)

3. 使用sklearn 中的train_test_split划分训练集和测试集

X_train, X_test, y_train, y_test = train_test_split(X, new_labels, random_state=42, test_size=0.2)

4. 训练方式:

model.fit(X_train, y_train, batch_size=32, epochs=100, validation_split=0.3)

方式二 : 使用keras 的 ImageDataGenerator分批加载数据

1.优点 :
分批加载数据
2.缺点 :
需要对数据进行按标签分文件夹
3.适用文件存储方式 :
图片根据标签对数据分文件夹

4.实现方式：

1. 创建文件夹

base_dir = 'E:/mldata/dogvscat/cats_and_dogs_small'
os.mkdir(base_dir)
train_dir = os.path.join(base_dir, 'train')
os.mkdir(train_dir)
validation_dir = os.path.join(base_dir, 'validation')
os.mkdir(validation_dir)
test_dir = os.path.join(base_dir, 'test')
os.mkdir(test_dir)
train_cats_dir = os.path.join(train_dir, 'cats')
os.mkdir(train_cats_dir)
train_dogs_dir = os.path.join(train_dir, 'dogs')
os.mkdir(train_dogs_dir)
validation_cats_dir = os.path.join(validation_dir, 'cats')
os.mkdir(validation_cats_dir)
validation_dogs_dir = os.path.join(validation_dir, 'dogs')
os.mkdir(validation_dogs_dir)
test_cats_dir = os.path.join(test_dir, 'cats')
os.mkdir(test_cats_dir)
test_dogs_dir = os.path.join(test_dir, 'dogs')
os.mkdir(test_dogs_dir)

2. 复制文件到文件夹

fnames = ['cat.{}.jpg'.format(i) for i in range(1000)]
for fname in fnames:
   src = os.path.join(original_dataset_dir, fname)
   dst = os.path.join(train_cats_dir, fname)
   shutil.copyfile(src, dst)
fnames = ['cat.{}.jpg'.format(i) for i in range(1000, 1500)]
for fname in fnames:
   src = os.path.join(original_dataset_dir, fname)
   dst = os.path.join(validation_cats_dir, fname)
   shutil.copyfile(src, dst)
fnames = ['cat.{}.jpg'.format(i) for i in range(1500, 2000)]
for fname in fnames:
   src = os.path.join(original_dataset_dir, fname)
   dst = os.path.join(test_cats_dir, fname)
   shutil.copyfile(src, dst)
fnames = ['dog.{}.jpg'.format(i) for i in range(1000)]
for fname in fnames:
   src = os.path.join(original_dataset_dir, fname)
   dst = os.path.join(train_dogs_dir, fname)
   shutil.copyfile(src, dst)
fnames = ['dog.{}.jpg'.format(i) for i in range(1000, 1500)]
for fname in fnames:
 src = os.path.join(original_dataset_dir, fname)
   dst = os.path.join(validation_dogs_dir, fname)
   shutil.copyfile(src, dst)
fnames = ['dog.{}.jpg'.format(i) for i in range(1500, 2000)]
for fname in fnames:
   src = os.path.join(original_dataset_dir, fname)
   dst = os.path.join(test_dogs_dir, fname)
   shutil.copyfile(src, dst)

3. 使用 ImageDataGenerator 加载数据

from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale=1./255)
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory(
 train_dir,
 target_size=(150, 150),
 batch_size=20,
 class_mode='binary')
validation_generator = test_datagen.flow_from_directory(
 validation_dir,
 target_size=(150, 150),
 batch_size=20,
 class_mode='binary')

4. 训练方式

history = model.fit_generator(
 train_generator,
 steps_per_epoch=100,
 epochs=30,
 validation_data=validation_generator,
 validation_steps=50)

方式三 :通过pandas加载数据

labels_needed = ['Bombay', 'Calico', 'Burmese', 'Himalayan', 'Munchkin', 'Ragdoll', 'Siberian', 'British Shorthair',
                 'Russian Blue',
                 'Dilute Calico']
Filepaths = []
labels = []

for image_file in file_list:
    label = image_file.split(os.path.sep)[-2]
    if label in labels_needed:
        Filepaths.append(image_file)
        labels.append(label)

# 将数据转化成dataframe
df = pd.DataFrame(list(zip(Filepaths, labels)), columns=['Filepath', 'Labels'])
# 将数据转打乱
df = (df.sample(frac=1).reset_index()).drop(columns='index')

train_ratio = .75
validation_ratio = 0.10
test_ratio = 0.25
# 划分数据集
train, test = train_test_split(df, test_size=test_ratio)
val, test = train_test_split(test, test_size=test_ratio / (test_ratio + validation_ratio))

image_datagen = ImageDataGenerator(rescale=1. / 255,
                                   rotation_range=30,
                                   width_shift_range=0.2,
                                   height_shift_range=0.2,
                                   horizontal_flip='true')
# 加载train数据
x_train = image_datagen.flow_from_dataframe(dataframe=train, x_col='Filepath', y_col='Labels', target_size=(299, 299),
                                            shuffle=False, batch_size=10, seed=10)
# 加载val数据
x_val = image_datagen.flow_from_dataframe(dataframe=val, x_col='Filepath', y_col='Labels', target_size=(299, 299),
                                          shuffle=False, batch_size=10, seed=10)
# 加载test数据
x_test = image_datagen.flow_from_dataframe(dataframe=test, x_col='Filepath', y_col='Labels', target_size=(299, 299),
                                           shuffle=False, batch_size=10, seed=10)