我尝试使用Tensorflow 1.4.0尝试对原始记录进行分类。该过程如下。
拳头：读取图像和标签，并将" tfrecord"格式输出到fiiles中。第二：阅读TF记录和培训
写tfrecord脚本是
!/usr/bin/env python3
#coding:utf-8
import argparse
import os
import random
import numpy as np
from PIL import Image
import tensorflow as tf
def make_example(label_index, image):
return tf.train.Example(features = tf.train.Features(feature={
'label_index': tf.train.Feature(int64_list=tf.train.Int64List(value=[label_index])),
'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image]))
}))
def write_tfrecord(dataset, outputfilepath):
writer = tf.python_io.TFRecordWriter(outputfilepath)
for label_of_one_hot, image in dataset:
ex = make_example(label_of_one_hot, image)
writer.write(ex.SerializeToString())
writer.close()
def importingargs():
parser = argparse.ArgumentParser("tensorflow exampe")
parser.add_argument("--datafolderpath", "-df", help="datafolderpath")
parser.add_argument("--filepath", "-f", help="filepath", required=True)
parser.add_argument("--labelfilepath", "-lf", help="label filepath")
parser.add_argument("--outputfolderpath", "-of", help="outputfolderpath of tf records")
parser.add_argument("--seed", "-s", type=int, required=False, default=0)
args = parser.parse_args()
return args.filepath, args.datafolderpath, args.labelfilepath, args.outputfolderpath, args.seed
def load_data(filepath, datafolderpath, labelfilepath):
with open(labelfilepath, "r") as rf:
labellist = [ line.strip() for line in rf.readlines() ]
with open(filepath, "r") as rf:
filepathlist = [ line.strip() for line in rf.readlines() ]
alldatasets = list()
for filepath in filepathlist:
imagefilepath = os.path.join(datafolderpath, filepath)
# image = open(imagefilepath).read()
img_obj = Image.open(imagefilepath).convert("L")
img = np.array(img_obj)
w, h = img.shape
print(w, h)
print(w*h)
img = img.reshape(w*h).tostring()
print(type(img))
filename = filepath.split(os.path.sep)[-1]
label = filename.split(".")[0].split("_")[1]
index = labellist.index(label) +1
print(index)
alldatasets.append([ index, img ])
return alldatasets
def splitdata(datasets):
random.shuffle(datasets)
train_indexes = [ 0, int(len(datasets) * 0.8 ) ]
valid_indexes = [ train_indexes[-1], int(len(datasets) * 0.9 ) ]
test_indexes = [ valid_indexes[-1], int(len(datasets)) ]
train_data = datasets[train_indexes[0]:train_indexes[1]]
valid_data = datasets[valid_indexes[0]:valid_indexes[1]]
test_data = datasets[test_indexes[0]:test_indexes[1]]
print("train num: %d" % len(train_data))
print("test num: %d" % len(test_data))
print("valid num: %d" % len(valid_data))
return train_data, valid_data, test_data
def main():
filepath, datafolderpath, labelfilepath, outputfolderpath, seed = importingargs()
random.seed(seed)
alldatasets = load_data(filepath, datafolderpath, labelfilepath)
train_data, valid_data, test_data = splitdata(alldatasets)
train_outputfilepath = os.path.join(outputfolderpath, "train.tfrecord")
valid_outputfilepath = os.path.join(outputfolderpath, "valid.tfrecord")
test_outptufilepath = os.path.join(outputfolderpath, "test.tfrecord")
write_tfrecord(train_data, train_outputfilepath)
write_tfrecord(valid_data, valid_outputfilepath)
write_tfrecord(test_data, test_outptufilepath)
if __name__ == "__main__":
main()
load_dataset文件导入train.py
#!/usr/bin/env python3
#coding:utf-8
import argparse
import os
import numpy as np
from PIL import Image
import tensorflow as tf
def read_tfrecord(inputfilepath):
print("read record")
reader = tf.TFRecordReader()
filename_que = tf.train.string_input_producer([inputfilepath])
key, value = reader.read(filename_que)
features = tf.parse_single_example(value,features = {
'label_index': tf.FixedLenFeature([], tf.string),
'image': tf.FixedLenFeature([], tf.string)
})
images = tf.decode_raw(features['image'], tf.float32)
images.set_shape([32*32])
images = tf.cast(images, tf.float32) * (1. / 255)
# images = tf.reshape(images, [-1])
labels = tf.decode_raw(features['label_index'], tf.int32)
# labels = tf.cast(features['label_index'], tf.int64)
# labels.set_shape([5])
print("call one hot")
label_index_one_hot = tf.one_hot(labels, 5)
label_index_one_hot.set_shape([5])
return images, label_index_one_hot
培训脚本是
import os
import random
import tensorflow as tf
import load_datasets
import datasets
import make_datasets
print("def input and output")
images = tf.placeholder(tf.float32, shape=[None, 32*32])
labels = tf.placeholder(tf.int32, shape=[None, 5])
print("def layers")
x = tf.placeholder(tf.float32, [ None, 32*32 ])
y_ = tf.placeholder(tf.float32, [None, 5 ])
# W1 = tf.Variable(tf.zeros([ 32*32, 500 ]))
# b1 = tf.Variable(tf.zeros([ 500 ]))
# W2 = tf.Variable(tf.zeros([ 500, 5 ]))
# b2 = tf.Variable(tf.zeros([ 5 ]))
print("def function")
# h1 = tf.matmul(x, W1) + b1
# y = tf.matmul(h1, W2) + b2
W = tf.Variable(tf.zeros([ 32*32, 5 ]))
b = tf.Variable(tf.zeros([ 5 ]))
y = tf.matmul(x, W) + b
print("def leraning model")
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=y))
train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)
correct_prediction= tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print("load train dataset")
trainfilepath = "../03tfrecords/train.tfrecord"
images, labels = load_datasets.read_tfrecord(trainfilepath)
input_queue = tf.train.slice_input_producer( [images, labels ], num_epochs=10, shuffle=False )
image_batch, label_batch = tf.train.batch( [images, labels], batch_size=10)
print("load test dataset")
testfilepath = "../03tfrecords/test.tfrecord"
test_image, test_label = load_datasets.read_tfrecord(testfilepath)
img_test_batch, label_test_batch = tf.train.batch([test_image,test_label],batch_size=16)
with tf.Session() as sess:
print("init layer value")
sess.run(tf.global_variables_initializer())
print("start training")
tf.train.start_queue_runners(sess)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
while not coord.should_stop():
for i in range(0, 10):
print("train num %d" % (i+1))
imgs, labels = sess.run([image_batch, label_batch])
sess.run(train_step, feed_dict={x:imgs, y_: labels})
imgs_test, labels_text = sess.run([img_test_batch, label_test_batch])
print(sess.run(accuracy, feed_dict={x:imgs_test, y_:labels_text}))
finally:
coord.request_stop()
coord.join(threads)
制作tfrecords效果很好，但是在训练脚本中，发生了错误。
Traceback (most recent call last):
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 576, in merge_with
self.assert_same_rank(other)
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 621, in assert_same_rank
other))
ValueError: Shapes (?, 5) and (5,) must have the same rank
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "train.py", line 45, in <module>
images, labels = load_datasets.read_tfrecord(trainfilepath)
File "/home/omori/tensorflow_example/01src/load_datasets.py", line 30, in read_tfrecord
label_index_one_hot.set_shape([5])
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 407, in set_shape
self._shape = self._shape.merge_with(shape)
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 582, in merge_with
raise ValueError("Shapes %s and %s are not compatible" % (self, other))
ValueError: Shapes (?, 5) and (5,) are not compatible
我搜索了许多网站，但我无法获得解决方案。我该如何解决？
decode_raw：
Returns:
A Tensor of type out_type. A Tensor with one more dimension than the input bytes. The added
dimension will have size equal to the length of the elements of bytes divided by the number
of bytes to represent out_type.
所以在您的read_tfrecord功能中
labels = tf.decode_raw(features['label_index'], tf.int32)
给出labels超大维度。您可以使用
label_index_one_hot = tf.one_hot(labels[0], 5)
（注意添加的[0]）
我必须承认我不明白增加的维度是什么。