mnist/parse_data.py at master · angcbi/mnist · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import struct

import numpy as np
from matplotlib import pyplot as plt

train_x_file = './source_mnist/train-images.idx3-ubyte'
train_y_file = './source_mnist/train-labels.idx1-ubyte'
test_x_file = './source_mnist/t10k-images.idx3-ubyte'
text_y_file = './source_mnist/t10k-labels.idx1-ubyte'

np_train_x_file = './data/train'
np_train_y_file = './data/train_label'
np_test_x_file = './data/test'
np_test_y_file = './data/test_label'


# parse train file
def parse_train_x_file():
    with open(train_x_file, 'rb') as file:
        desc_format = struct.Struct('>iiii')
        data_format = struct.Struct('>784B')

        # parse 16 byte
        magic, total, rows, cols = desc_format.unpack_from(file.read(desc_format.size))
        print(magic, total, rows, cols)

        # parse data
        mnist_list = []
        for i in range(total):
            # parse 784 byte
            read_data = file.read(data_format.size)
            data = data_format.unpack_from(read_data)
            mnist_list.append(data)

        train_x = np.array(mnist_list)
        np.save(np_train_x_file, train_x)

        # show first image, reshape 28*28
        fig = plt.figure()
        plt.imshow(train_x[0].reshape(28, 28), cmap='autumn')
        plt.show()


def parse_train_y_file():
    with open(train_y_file, 'rb') as file:
        desc_format = struct.Struct('>ii')
        magic, total = desc_format.unpack_from(file.read(desc_format.size))
        print(magic, total)

        data_format = struct.Struct('>%sB' % total)
        train_y = np.array(data_format.unpack_from(file.read(data_format.size)))
        np.save(np_train_y_file, train_y)


def parse_test_x_file():
    with open(test_x_file, 'rb') as file:
        desc_format = struct.Struct('>iiii')
        data_format = struct.Struct('>784B')

        # parse 16 byte
        magic, total, rows, cols = desc_format.unpack_from(file.read(desc_format.size))
        print(magic, total, rows, cols)

        # parse data
        mnist_list = []
        for i in range(total):
            # parse 784 byte
            read_data = file.read(data_format.size)
            data = data_format.unpack_from(read_data)
            mnist_list.append(data)

        train_x = np.array(mnist_list)
        np.save(np_test_x_file, train_x)

        # show first image, reshape 28*28
        fig = plt.figure()
        plt.imshow(train_x[0].reshape(28, 28), cmap='autumn')
        plt.show()


def parse_test_y_file():
    with open(text_y_file, 'rb') as file:
        desc_format = struct.Struct('>ii')
        magic, total = desc_format.unpack_from(file.read(desc_format.size))
        print(magic, total)

        data_format = struct.Struct('>%sB' % total)
        train_y = np.array(data_format.unpack_from(file.read(data_format.size)))
        np.save(np_test_y_file, train_y)


if __name__ == '__main__':
    parse_train_x_file()
    parse_train_y_file()
    parse_test_x_file()
    parse_test_y_file()