Tensorflow2.0学习(六):猫狗大战1、制作与读取record数据

112 阅读 0 评论 74 点赞

我是靠谱客的博主忧郁龙猫，这篇文章主要介绍Tensorflow2.0学习(六):猫狗大战1、制作与读取record数据，现在分享给大家，希望可以做个参考。

一、 TFRecords数据介绍:

TFRecords可以允许你讲任意的数据转换为TensorFlow所支持的格式，这种方法可以使TensorFlow的数据集更容易与网络应用架构相匹配。这种建议的方法就是使用TFRecords文件，TFRecords文件包含了[tf.train.Example 协议内存块(protocol buffer)](协议内存块包含了字段[Features]。你可以写一段代码获取你的数据，将数据填入到Example协议内存块(protocol buffer)，将协议内存块序列化为一个字符串，并且通过[tf.python_io.TFRecordWriter class]写入到TFRecords文件。

TFRecords文件格式在图像识别中有很好的使用,其可以将二进制数据和标签数据(训练的类别标签)数据存储在同一个文件中,它可以在模型进行训练之前通过预处理步骤将图像转换为TFRecords格式,此格式最大的优点实践每幅输入图像和与之关联的标签放在同一个文件中.TFRecords文件是一种二进制文件,其不对数据进行压缩,所以可以被快速加载到内存中.格式不支持随机访问，因此它适合于大量的数据流，但不适用于快速分片或其他非连续存取。

二、利用猫狗大战数据集制作record数据

首先,生成一个txt文件(caffe的习惯,将图像路径和对应的标签写写入到txt文件中)
txt文件内容如下:

复制代码

1
2
3
4
/path/DogVsCat/0/cat.1337.jpg 0
/path/DogVsCat/0/cat.12044.jpg 0
/path/DogVsCat/0/dog.6658.jpg 1

生成Record的代码如下:

复制代码

import tensorflow as tf
import numpy as np
import random

def int64_feature(value):
  """Wrapper for inserting int64 features into Example proto."""
  if not isinstance(value, list):
    value = [value]
  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
 
def bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def imgLabel2example(img_path, classes):
    with tf.io.gfile.GFile(img_path, 'rb') as fid:
        encoded_jpg = fid.read()
    example = tf.train.Example(features=tf.train.Features(feature={
        'image/filename': bytes_feature(img_path.encode('utf8')),
        'image/encoded': bytes_feature(encoded_jpg),
        'image/class': int64_feature(int(classes)),
    }))
    
    return example

trainRecordPath = "./train.record" # 训练集的record数据存储的位置
testRecordPath = "./test.record"   # 测试集的record数据存储的位置
writerTrain = tf.io.TFRecordWriter(trainRecordPath)  # 打开并写入
writerTest = tf.io.TFRecordWriter(testRecordPath)

txt_path = "./DogVsCat/train.txt" # 存放图像位置和对应label的txt文件
data_lib = open(txt_path, "r").readlines() 
random.shuffle(data_lib)

trainLib = data_lib[: int(len(data_lib)*0.9)]
testLib = data_lib[int(len(data_lib)*0.9) : ]

for line in testLib:
    img_path = line.strip().split(" ")[0]
    label = int(line.strip().split(" ")[1]) 
    # 生成tf.train.feature属性(key和value的键值对)，在将这些单独feature整合成features
    # 生成Example并序列化
    example = imgLabel2example(img_path, label) 
    # 将example序列化，压缩以减少size
    serialized_example = example.SerializeToString()
    writerTest.write(serialized_example)
writerTest.close()

for line in trainLib:
    img_path = line.strip().split(" ")[0]
    label = int(line.strip().split(" ")[1]) 
    # 生成tf.train.feature属性(key和value的键值对)，在将这些单独feature整合成features
    # 生成Example并序列化
    examples = imgLabel2example(img_path, label) 
    # 将example序列化，压缩以减少size
    serialized_example = examples.SerializeToString()
    writerTrain.write(serialized_example)
    
writerTrain.close()
print("sess write")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import tensorflow as tf
import numpy as np
import random

def int64_feature(value):
  """Wrapper for inserting int64 features into Example proto."""
  if not isinstance(value, list):
    value = [value]
  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
 
def bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def imgLabel2example(img_path, classes):
    with tf.io.gfile.GFile(img_path, 'rb') as fid:
        encoded_jpg = fid.read()
    example = tf.train.Example(features=tf.train.Features(feature={
        'image/filename': bytes_feature(img_path.encode('utf8')),
        'image/encoded': bytes_feature(encoded_jpg),
        'image/class': int64_feature(int(classes)),
    }))
    
    return example

trainRecordPath = "./train.record" # 训练集的record数据存储的位置
testRecordPath = "./test.record"   # 测试集的record数据存储的位置
writerTrain = tf.io.TFRecordWriter(trainRecordPath)  # 打开并写入
writerTest = tf.io.TFRecordWriter(testRecordPath)

txt_path = "./DogVsCat/train.txt" # 存放图像位置和对应label的txt文件
data_lib = open(txt_path, "r").readlines() 
random.shuffle(data_lib) 

trainLib = data_lib[: int(len(data_lib)*0.9)]
testLib = data_lib[int(len(data_lib)*0.9) : ]

for line in testLib:
    img_path = line.strip().split(" ")[0]
    label = int(line.strip().split(" ")[1]) 
    # 生成tf.train.feature属性(key和value的键值对)，在将这些单独feature整合成features
    # 生成Example并序列化
    example = imgLabel2example(img_path, label) 
    # 将example序列化，压缩以减少size
    serialized_example = example.SerializeToString()
    writerTest.write(serialized_example)
writerTest.close()

for line in trainLib:
    img_path = line.strip().split(" ")[0]
    label = int(line.strip().split(" ")[1]) 
    # 生成tf.train.feature属性(key和value的键值对)，在将这些单独feature整合成features
    # 生成Example并序列化
    examples = imgLabel2example(img_path, label) 
    # 将example序列化，压缩以减少size
    serialized_example = examples.SerializeToString()
    writerTrain.write(serialized_example)
    
writerTrain.close()
print("sess write")

三、读取Record,并显示图像

复制代码

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

record_path = "./train.record"

# 调用后我们会得到一个Dataset(tf.data.Dataset)，字面理解，这里面就存放着我们之前写入的所有Example。
dataset = tf.data.TFRecordDataset(record_path)
#定义一个解析函数

feature_description = {
    'image/filename' : tf.io.FixedLenFeature([], tf.string),
    'image/class': tf.io.FixedLenFeature([] , tf.int64),
    'image/encoded' : tf.io.FixedLenFeature([], tf.string)
}

def parese_example(serialized_example):
    feature_dict = tf.io.parse_single_example (serialized_example, feature_description)
    image = tf.io.decode_jpeg(feature_dict['image/encoded'])    # 解码JPEG图片
    image = tf.image.resize_with_crop_or_pad(image, 224, 224)
    image = tf.cast(image, tf.float32)
    
    feature_dict['image'] = image
    return feature_dict['image'], feature_dict['image/class']

dataset = dataset.map(parese_example)
dataset = dataset.repeat().shuffle(5000).batch(1).prefetch(1)

for img, label in dataset.take(1): # 只取前1条
    print(label.numpy())
    print(img.numpy()[0].shape)
    plt.imshow(np.array(img.numpy()[0], np.uint8))
    plt.show()
#     print (np.frombuffer(row['image/class'].numpy(), dtype=np.uint8)) # 如果要恢复成3d数组，可reshape

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

record_path = "./train.record"

# 调用后我们会得到一个Dataset(tf.data.Dataset)，字面理解，这里面就存放着我们之前写入的所有Example。
dataset = tf.data.TFRecordDataset(record_path)
#定义一个解析函数

feature_description = {
    'image/filename' : tf.io.FixedLenFeature([], tf.string),
    'image/class': tf.io.FixedLenFeature([] , tf.int64),
    'image/encoded' : tf.io.FixedLenFeature([], tf.string)
}

def parese_example(serialized_example):
    feature_dict = tf.io.parse_single_example (serialized_example, feature_description)
    image = tf.io.decode_jpeg(feature_dict['image/encoded'])    # 解码JPEG图片
    image = tf.image.resize_with_crop_or_pad(image, 224, 224)
    image = tf.cast(image, tf.float32)
    
    feature_dict['image'] = image
    return feature_dict['image'], feature_dict['image/class']

dataset = dataset.map(parese_example)
dataset = dataset.repeat().shuffle(5000).batch(1).prefetch(1)

for img, label in dataset.take(1): # 只取前1条
    print(label.numpy())
    print(img.numpy()[0].shape)
    plt.imshow(np.array(img.numpy()[0], np.uint8))
    plt.show()
#     print (np.frombuffer(row['image/class'].numpy(), dtype=np.uint8)) # 如果要恢复成3d数组，可reshape