我正在使用张量流模型，object_detectioncreate_pascal_tf_record.py重命名create_face_tf_record.pywider_face数据集转换为TF-Record：
D: -STUDYmodelsresearch>python object_detectiondataset_toolscreate_face_tf_record.py
--data_dir=D:/0-STUDY
--year=widerface
--output_path=D: -STUDYdatasetswiderfaceTF_datatrain.record
--set=train
包装只是为了看起来不错的
输出：
2020-02-11 09:41:46.804523: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_100.dll
WARNING:tensorflow:From object_detectiondataset_toolscreate_face_tf_record.py:189: The name tf.app.run is deprecated. Please use tf.compat.v1.app.run instead.
WARNING:tensorflow:From object_detectiondataset_toolscreate_face_tf_record.py:163: The name tf.python_io.TFRecordWriter is deprecated. Please use tf.io.TFRecordWriter instead.
W0211 09:41:49.443757 16972 module_wrapper.py:139] From object_detectiondataset_toolscreate_face_tf_record.py:163: The name tf.python_io.TFRecordWriter is deprecated. Please use tf.io.TFRecordWriter instead.
WARNING:tensorflow:From D: -STUDYmodelsresearchobject_detectionutilslabel_map_util.py:138: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.
W0211 09:41:49.445752 16972 module_wrapper.py:139] From D: -STUDYmodelsresearchobject_detectionutilslabel_map_util.py:138: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.
I0211 09:41:49.448744 16972 create_face_tf_record.py:168] Reading from PASCAL widerface dataset.
I0211 09:41:49.491163 16972 create_face_tf_record.py:175] On image 0 of 12880
D: -STUDYmodelsresearchobject_detectionutilsdataset_util.py:79: FutureWarning: The behavior of this method will change in future versions. Use specific 'len(elem)' or 'elem is not None' test instead.
if not xml:
Traceback (most recent call last):
File "object_detectiondataset_toolscreate_face_tf_record.py", line 189, in <module>
tf.app.run()
File "C:ProgramDataAnaconda3libsite-packagestensorflow_corepythonplatformapp.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "C:ProgramDataAnaconda3libsite-packagesabslapp.py", line 299, in run
_run_main(main, args)
File "C:ProgramDataAnaconda3libsite-packagesabslapp.py", line 250, in _run_main
sys.exit(main(argv))
File "object_detectiondataset_toolscreate_face_tf_record.py", line 182, in main
tf_example = dict_to_tf_example(data, FLAGS.data_dir, label_map_dict,FLAGS.ignore_difficult_instances)
File "object_detectiondataset_toolscreate_face_tf_record.py", line 125, in dict_to_tf_example
classes.append(label_map_dict[obj['name']])
KeyError: 'face'
树：
D: -STUDY> tree -L 2
.
├── datasets
│ └── widerface
| └── TF_data
├── models
│ ├── AUTHORS
│ ├── CODEOWNERS
│ ├── CONTRIBUTING.md
│ ├── ISSUE_TEMPLATE.md
│ ├── LICENSE
│ ├── README.md
│ ├── WORKSPACE
│ ├── models.zip
│ ├── official
│ ├── research
│ ├── samples
│ ├── tutorials
│ └── widerface
└── widerface
├── Annotations
├── ImageSets
├── JPEGImages
├── WIDER_test
├── WIDER_train
├── WIDER_val
└── wider_face_split
create_face_tf_record.py：
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
r"""Convert raw PASCAL dataset to TFRecord for object_detection.
Example usage:
python object_detection/dataset_tools/create_pascal_tf_record.py
--data_dir=/home/user/VOCdevkit
--year=VOC2012
--output_path=/home/user/pascal.record
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import hashlib
import io
import logging
import os
from lxml import etree
import PIL.Image
import tensorflow as tf
import pdb
from object_detection.utils import dataset_util
from object_detection.utils import label_map_util
flags = tf.app.flags
flags.DEFINE_string('data_dir', '', 'Root directory to raw PASCAL VOC dataset.')
flags.DEFINE_string('set', 'train', 'Convert training set, validation set or '
'merged set.')
flags.DEFINE_string('annotations_dir', 'Annotations',
'(Relative) path to annotations directory.')
flags.DEFINE_string('year', 'VOC2007', 'Desired challenge year.')
flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
flags.DEFINE_string('label_map_path', 'object_detection/data/pascal_label_map.pbtxt',
'Path to label map proto')
flags.DEFINE_boolean('ignore_difficult_instances', False, 'Whether to ignore '
'difficult instances')
FLAGS = flags.FLAGS
SETS = ['train', 'val', 'trainval', 'test']
YEARS = ['fddb', 'widerface'] # ------------------1️⃣
def dict_to_tf_example(data,
dataset_directory,
label_map_dict,
ignore_difficult_instances=False,
image_subdirectory='JPEGImages'):
"""Convert XML derived dict to tf.Example proto.
Notice that this function normalizes the bounding box coordinates provided
by the raw data.
Args:
data: dict holding PASCAL XML fields for a single image (obtained by
running dataset_util.recursive_parse_xml_to_dict)
dataset_directory: Path to root directory holding PASCAL dataset
label_map_dict: A map from string label names to integers ids.
ignore_difficult_instances: Whether to skip difficult instances in the
dataset (default: False).
image_subdirectory: String specifying subdirectory within the
PASCAL dataset directory holding the actual image data.
Returns:
example: The converted tf.Example.
Raises:
ValueError: if the image pointed to by data['filename'] is not a valid JPEG
"""
img_path = os.path.join(data['folder'], image_subdirectory, data['filename'])
full_path = os.path.join(dataset_directory, img_path)
with tf.gfile.GFile(full_path, 'rb') as fid:
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = PIL.Image.open(encoded_jpg_io)
if image.format != 'JPEG':
raise ValueError('Image format not JPEG')
key = hashlib.sha256(encoded_jpg).hexdigest()
width = int(data['size']['width'])
height = int(data['size']['height'])
xmin = []
ymin = []
xmax = []
ymax = []
classes = []
classes_text = []
truncated = []
poses = []
difficult_obj = []
if 'object' in data:
for obj in data['object']:
difficult = bool(int(obj['difficult']))
if ignore_difficult_instances and difficult:
continue
difficult_obj.append(int(difficult))
xmin.append(float(obj['bndbox']['xmin']) / width)
ymin.append(float(obj['bndbox']['ymin']) / height)
xmax.append(float(obj['bndbox']['xmax']) / width)
ymax.append(float(obj['bndbox']['ymax']) / height)
classes_text.append(obj['name'].encode('utf8'))
classes.append(label_map_dict[obj['name']])
truncated.append(int(obj['truncated']))
poses.append(obj['pose'].encode('utf8'))
example = tf.train.Example(features=tf.train.Features(feature={
'image/height': dataset_util.int64_feature(height),
'image/width': dataset_util.int64_feature(width),
'image/filename': dataset_util.bytes_feature(
data['filename'].encode('utf8')),
'image/source_id': dataset_util.bytes_feature(
data['filename'].encode('utf8')),
'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')),
'image/encoded': dataset_util.bytes_feature(encoded_jpg),
'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
'image/object/bbox/xmin': dataset_util.float_list_feature(xmin),
'image/object/bbox/xmax': dataset_util.float_list_feature(xmax),
'image/object/bbox/ymin': dataset_util.float_list_feature(ymin),
'image/object/bbox/ymax': dataset_util.float_list_feature(ymax),
'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
'image/object/class/label': dataset_util.int64_list_feature(classes),
'image/object/difficult': dataset_util.int64_list_feature(difficult_obj),
'image/object/truncated': dataset_util.int64_list_feature(truncated),
'image/object/view': dataset_util.bytes_list_feature(poses),
}))
return example
def main(_):
if FLAGS.set not in SETS:
raise ValueError('set must be in : {}'.format(SETS))
if FLAGS.year not in YEARS:
raise ValueError('year must be in : {}'.format(YEARS))
data_dir = FLAGS.data_dir
years = ['fddb', 'widerface'] # ------------------2️⃣
if FLAGS.year != 'merged':
years = [FLAGS.year]
writer = tf.python_io.TFRecordWriter(FLAGS.output_path)
label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)
for year in years:
logging.info('Reading from PASCAL %s dataset.', year)
examples_path = os.path.join(data_dir, year, 'ImageSets', 'Main',
FLAGS.set + '.txt') # ------------------3️⃣
annotations_dir = os.path.join(data_dir, year, FLAGS.annotations_dir)
examples_list = dataset_util.read_examples_list(examples_path)
for idx, example in enumerate(examples_list):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(examples_list))
path = os.path.join(annotations_dir, example + '.xml')
with tf.gfile.GFile(path, 'r') as fid:
xml_str = fid.read()
xml = etree.fromstring(xml_str)
data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']
tf_example = dict_to_tf_example(data, FLAGS.data_dir, label_map_dict,FLAGS.ignore_difficult_instances)
writer.write(tf_example.SerializeToString())
writer.close()
if __name__ == '__main__':
tf.app.run()
1️2️⃣ ⃣3️⃣ 是与create_pascal_tf_record.py的区别
接下来我应该怎么做？ 🙃
您可以使用以下方法将宽面数据集转换为 TFRecords。
1.您需要创建一个config.py文件。
# Training
TRAIN_WIDER_PATH = "widerface/WIDER_train/"
#Validation
VAL_WIDER_PATH = "widerface/WIDER_val/"
#Testing
TEST_WIDER_PATH = "widerface/WIDER_test/"
# Ground Truth
GROUND_TRUTH_PATH = "widerface/wider_face_split/"
# Output
OUTPUT_PATH = "datasets/widerface/TF_data/"
- 用于生成 TFRecords(create_tf_record.py( 的代码。
这是代码：
import tensorflow as tf
import numpy
import cv2
import os
import hashlib
import config
from utils import dataset_util
def parse_test_example(f, images_path):
height = None # Image height
width = None # Image width
filename = None # Filename of the image. Empty if image is not from file
encoded_image_data = None # Encoded image bytes
image_format = b'jpeg' # b'jpeg' or b'png'
filename = f.readline().rstrip()
if not filename:
raise IOError()
filepath = os.path.join(images_path, filename)
image_raw = cv2.imread(filepath)
encoded_image_data = open(filepath, "rb").read()
key = hashlib.sha256(encoded_image_data).hexdigest()
height, width, channel = image_raw.shape
tf_example = tf.train.Example(features=tf.train.Features(feature={
'image/height': dataset_util.int64_feature(int(height)),
'image/width': dataset_util.int64_feature(int(width)),
'image/filename': dataset_util.bytes_feature(filename.encode('utf-8')),
'image/source_id': dataset_util.bytes_feature(filename.encode('utf-8')),
'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')),
'image/encoded': dataset_util.bytes_feature(encoded_image_data),
'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
}))
return tf_example
def parse_example(f, images_path):
height = None # Image height
width = None # Image width
filename = None # Filename of the image. Empty if image is not from file
encoded_image_data = None # Encoded image bytes
image_format = b'jpeg' # b'jpeg' or b'png'
xmins = [] # List of normalized left x coordinates in bounding box (1 per box)
xmaxs = [] # List of normalized right x coordinates in bounding box (1 per box)
ymins = [] # List of normalized top y coordinates in bounding box (1 per box)
ymaxs = [] # List of normalized bottom y coordinates in bounding box (1 per box)
classes_text = [] # List of string class name of bounding box (1 per box)
classes = [] # List of integer class id of bounding box (1 per box)
poses = []
truncated = []
difficult_obj = []
filename = f.readline().rstrip()
if not filename:
raise IOError()
filepath = os.path.join(images_path, filename)
image_raw = cv2.imread(filepath)
encoded_image_data = open(filepath, "rb").read()
key = hashlib.sha256(encoded_image_data).hexdigest()
height, width, channel = image_raw.shape
face_num = int(f.readline().rstrip())
if not face_num:
face_num += 1
# raise Exception()
for i in range(face_num):
annot = f.readline().rstrip().split()
if not annot:
raise Exception()
# WIDER FACE DATASET CONTAINS SOME ANNOTATIONS WHAT EXCEEDS THE IMAGE BOUNDARY
if(float(annot[2]) > 25.0):
if(float(annot[3]) > 30.0):
xmins.append( max(0.005, (float(annot[0]) / width) ) )
ymins.append( max(0.005, (float(annot[1]) / height) ) )
xmaxs.append( min(0.995, ((float(annot[0]) + float(annot[2])) / width) ) )
ymaxs.append( min(0.995, ((float(annot[1]) + float(annot[3])) / height) ) )
classes_text.append(b'face')
classes.append(1)
poses.append("front".encode('utf8'))
truncated.append(int(0))
tf_example = tf.train.Example(features=tf.train.Features(feature={
'image/height': dataset_util.int64_feature(int(height)),
'image/width': dataset_util.int64_feature(int(width)),
'image/filename': dataset_util.bytes_feature(filename.encode('utf-8')),
'image/source_id': dataset_util.bytes_feature(filename.encode('utf-8')),
'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')),
'image/encoded': dataset_util.bytes_feature(encoded_image_data),
'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
'image/object/class/label': dataset_util.int64_list_feature(classes),
'image/object/difficult': dataset_util.int64_list_feature(int(0)),
'image/object/truncated': dataset_util.int64_list_feature(truncated),
'image/object/view': dataset_util.bytes_list_feature(poses),
}))
return tf_example
def run(images_path, description_file, output_path, no_bbox=False):
f = open(description_file)
writer = tf.python_io.TFRecordWriter(output_path)
i = 0
print("Processing {}".format(images_path))
while True:
try:
if no_bbox:
tf_example = parse_test_example(f, images_path)
else:
tf_example = parse_example(f, images_path)
writer.write(tf_example.SerializeToString())
i += 1
except IOError:
break
except Exception:
raise
writer.close()
print("Correctly created record for {} imagesn".format(i))
def main(unused_argv):
# Training
if config.TRAIN_WIDER_PATH is not None:
images_path = os.path.join(config.TRAIN_WIDER_PATH, "images")
description_file = os.path.join(config.GROUND_TRUTH_PATH, "wider_face_train_bbx_gt.txt")
output_path = os.path.join(config.OUTPUT_PATH, "train.tfrecord")
run(images_path, description_file, output_path)
# Validation
if config.VAL_WIDER_PATH is not None:
images_path = os.path.join(config.VAL_WIDER_PATH, "images")
description_file = os.path.join(config.GROUND_TRUTH_PATH, "wider_face_val_bbx_gt.txt")
output_path = os.path.join(config.OUTPUT_PATH, "val.tfrecord")
run(images_path, description_file, output_path)
# Testing. This set does not contain bounding boxes, so the tfrecord will contain images only
if config.TEST_WIDER_PATH is not None:
images_path = os.path.join(config.TEST_WIDER_PATH, "images")
description_file = os.path.join(config.GROUND_TRUTH_PATH, "wider_face_test_filelist.txt")
output_path = os.path.join(config.OUTPUT_PATH, "test.tfrecord")
run(images_path, description_file, output_path, no_bbox=True)
if __name__ == '__main__':
tf.app.run()
运行create_tf_record.py以生成 TFRecord 文件。
python create_tf_record.py
希望这能回答你的问题，快乐学习！