From ecfcf9cdbaec0e476004df9a569db1ee1f636f0b Mon Sep 17 00:00:00 2001 From: hvy Date: Fri, 20 Jan 2017 07:04:03 +0000 Subject: [PATCH] Clean up code --- lib/__init__.py | 1 + models/VGG.py | 147 --------------------------------------------- models/__init__.py | 0 utils/__init__.py | 0 utils/imgutil.py | 48 --------------- visualize.py | 120 ++++++++++++++++++++++++------------ 6 files changed, 81 insertions(+), 235 deletions(-) delete mode 100644 models/VGG.py delete mode 100644 models/__init__.py delete mode 100644 utils/__init__.py delete mode 100644 utils/imgutil.py diff --git a/lib/__init__.py b/lib/__init__.py index e69de29..aed4fa3 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -0,0 +1 @@ +from .models import * diff --git a/models/VGG.py b/models/VGG.py deleted file mode 100644 index 7cb5ae1..0000000 --- a/models/VGG.py +++ /dev/null @@ -1,147 +0,0 @@ -import cupy -import chainer -import chainer.links as L -import chainer.functions as F -from chainer import Variable, cuda - - -class VGG(chainer.Chain): - - """Input dimensions are (224, 224).""" - - def __init__(self): - super().__init__( - conv1_1=L.Convolution2D(3, 64, 3, stride=1, pad=1), - conv1_2=L.Convolution2D(64, 64, 3, stride=1, pad=1), - conv2_1=L.Convolution2D(64, 128, 3, stride=1, pad=1), - conv2_2=L.Convolution2D(128, 128, 3, stride=1, pad=1), - conv3_1=L.Convolution2D(128, 256, 3, stride=1, pad=1), - conv3_2=L.Convolution2D(256, 256, 3, stride=1, pad=1), - conv3_3=L.Convolution2D(256, 256, 3, stride=1, pad=1), - conv4_1=L.Convolution2D(256, 512, 3, stride=1, pad=1), - conv4_2=L.Convolution2D(512, 512, 3, stride=1, pad=1), - conv4_3=L.Convolution2D(512, 512, 3, stride=1, pad=1), - conv5_1=L.Convolution2D(512, 512, 3, stride=1, pad=1), - conv5_2=L.Convolution2D(512, 512, 3, stride=1, pad=1), - conv5_3=L.Convolution2D(512, 512, 3, stride=1, pad=1), - fc6=L.Linear(25088, 4096), - fc7=L.Linear(4096, 4096), - fc8=L.Linear(4096, 1000) - ) - - self.conv_blocks = [ - [self.conv1_1, self.conv1_2], - [self.conv2_1, self.conv2_2], - [self.conv3_1, self.conv3_2, self.conv3_3], - [self.conv4_1, self.conv4_2, self.conv4_3], - [self.conv5_1, self.conv5_2, self.conv5_3] - ] - self.deconv_blocks= [] - - # Keep track of the pooling indices inside each function instance - self.mps = [F.MaxPooling2D(2, 2, use_cudnn=False) for _ in self.conv_blocks] - - def __call__(self, x, train=False): - - """Return a softmax probability distribution over predicted classes.""" - - # Convolutional layers - hs, _ = self.feature_map_activations(x) - h = hs[-1] - - # Fully connected layers - h = F.dropout(F.relu(self.fc6(h)), train=train) - h = F.dropout(F.relu(self.fc7(h)), train=train) - h = self.fc8(h) - - return F.softmax(h) - - def feature_map_activations(self, x): - - """Forward pass through the convolutional layers of the VGG returning - all of its intermediate feature map activations.""" - - hs = [] - pre_pooling_sizes = [] - - h = x - - for conv_block, mp in zip(self.conv_blocks, self.mps): - for conv in conv_block: - h = F.relu(conv(h)) - - pre_pooling_sizes.append(h.data.shape[2:]) - h = mp(h) - hs.append(h.data) - - return hs, pre_pooling_sizes - - def activations(self, x, layer_idx): - - """Return filter activations projected back to the input space, - e.g. RGB images with shape (n_layers, n_feature_maps, 3, 224, 224) - for a particula layer. The given layer index is expected to be - 1-based. - """ - - if x.data.shape[0] != 1: - raise TypeError('Visualization is only supported for a single \ - image at a time') - - self.check_add_deconv_layers() - hs, unpooling_sizes = self.feature_map_activations(x) - - activation_maps = [] - n_activation_maps = hs[layer_idx].shape[1] - - xp = self.xp - - for i in range(n_activation_maps): # For each channel - h = hs[layer_idx].copy() - - condition = xp.zeros_like(h) - condition[0][i] = 1 # Keep one feature map and zero all other - - h = Variable(xp.where(condition, h, xp.zeros_like(h))) - - for i in reversed(range(layer_idx+1)): - p = self.mps[i] - h = F.upsampling_2d(h, p.indexes, p.kh, p.sy, p.ph, unpooling_sizes[i]) - for deconv in reversed(self.deconv_blocks[i]): - h = deconv(F.relu(h)) - - activation_maps.append(h.data) - - return xp.concatenate(activation_maps) - - def check_add_deconv_layers(self, nobias=True): - - """Add a deconvolutional layer for each convolutional layer already - defined in the network.""" - - if len(self.deconv_blocks) == len(self.conv_blocks): - return - - for conv_block in self.conv_blocks: - deconv_block = [] - for conv in conv_block: - out_channels, in_channels, kh, kw = conv.W.data.shape - - if isinstance(conv.W.data, cupy.ndarray): - initialW = cupy.asnumpy(conv.W.data) - else: - initialW = conv.W.data - - deconv = L.Deconvolution2D(out_channels, in_channels, - (kh, kw), stride=conv.stride, - pad=conv.pad, - initialW=initialW, - nobias=nobias) - - if isinstance(conv.W.data, cupy.ndarray): - deconv.to_gpu() - - self.add_link('de{}'.format(conv.name), deconv) - deconv_block.append(deconv) - - self.deconv_blocks.append(deconv_block) diff --git a/models/__init__.py b/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils/imgutil.py b/utils/imgutil.py deleted file mode 100644 index 628361b..0000000 --- a/utils/imgutil.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import math -import numpy as np -# import cv2 as cv -import matplotlib -matplotlib.use('Agg') # Workaround to save images when running over ssh sessions -import matplotlib.pyplot as plt -import matplotlib.image as mpimg -from PIL import Image - - -def tile_ims(filename, directory): - """Load all images in the given directory and tile them into one.""" - ims = [mpimg.imread(os.path.join(directory, f)) for f in - sorted(os.listdir(directory))] - save_ims(filename, np.array(ims)) - - -def save_im(filename, im): - # h, w, c = im.shape - # cv.imwrite(filename, im) - im = Image.fromarray(im) - im.save(filename) - - -def save_ims(filename, ims): - n, h, w, c = ims.shape - - # Plot the images on a grid - rows = int(math.ceil(math.sqrt(n))) - cols = int(round(math.sqrt(n))) - - # Each subplot should have the same resolutions as the image dimensions - - # TODO: Consider proper heights and widths for the subplots - h = 64 - w = 64 - - fig, axes = plt.subplots(rows, cols, figsize=(h, w)) - fig.subplots_adjust(hspace=0, wspace=0) - - for i, ax in enumerate(axes.flat): - ax.axis('off') # Hide x, y axes completely - if i < n: - ax.imshow(ims[i]) - - plt.savefig(filename, bbox_inches='tight') - plt.clf() diff --git a/visualize.py b/visualize.py index ee16d0b..1ed9db5 100644 --- a/visualize.py +++ b/visualize.py @@ -1,12 +1,13 @@ import argparse import os +import math import cupy import numpy as np -from chainer import serializers, cuda -from chainer import Variable -from models.VGG import VGG -from utils import imgutil +import matplotlib.pyplot as plt +import matplotlib.image as mpimg from PIL import Image +from chainer import Variable, serializers, cuda +from lib.models import VGG def parse_args(): @@ -15,10 +16,46 @@ def parse_args(): parser.add_argument('--out-dirname', type=str, default='results') parser.add_argument('--image-filename', type=str, default='images/cat.jpg') parser.add_argument('--model-filename', type=str, default='VGG.model') - return parser.parse_args() +def save_im(filename, im): + im = np.rollaxis(im, 0, 3) # (c, h, w) -> (h, w, c) + im = Image.fromarray(im) + im.save(filename) + + +def save_ims(filename, ims, dpi=100, scale=0.5): + n, c, h, w = ims.shape + + rows = int(math.ceil(math.sqrt(n))) + cols = int(round(math.sqrt(n))) + + fig, axes = plt.subplots(rows, cols, figsize=(w*cols/dpi*scale, h*rows/dpi*scale), dpi=dpi) + + for i, ax in enumerate(axes.flat): + if i < n: + ax.imshow(ims[i].transpose((1, 2, 0))) + ax.set_xticks([]) + ax.set_yticks([]) + ax.axis('off') + + plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0.1, hspace=0.1) + plt.savefig(filename, dpi=dpi, bbox_inces='tight', transparent=True) + plt.clf() + plt.close() + + +def tile_ims(filename, directory): + + """Load all images in the given directory and tile them into one.""" + + ims = [mpimg.imread(os.path.join(directory, f)) for f in sorted(os.listdir(directory))] + ims = np.array(ims) + ims = ims.transpose((0, 3, 1, 2)) # (n, h, w, c) -> (n, c, h ,w) + save_ims(filename, ims) + + def read_im(filename): """Return a preprocessed (averaged and resized to VGG) sample image.""" @@ -34,35 +71,60 @@ def read_im(filename): return im -def get_activations(model, x, layer_idx): +def visualize_layer_activations(model, x, layer_idx): """Compute the activations for each feature map for the given layer for this particular image. Note that the input x should be a mini-batch of size one, i.e. a single image. """ - if model._device_id >= 0: # GPU + if model._device_id and model._device_id >= 0: # Using GPU x = cupy.array(x) a = model.activations(Variable(x), layer_idx) - if model._device_id >= 0: + if model._device_id and model._device_id >= 0: a = cupy.asnumpy(a) - # Center at 0 with std 0.1 + # Normalize and rescale to [0, 255] a -= a.mean() a /= (a.std() + 1e-5) a *= 0.1 + a -= a.min() + a /= a.max() + a *= 255 - # Clip to [0, 1] - a += 0.5 - a = np.clip(a, 0, 1) + return a.astype(np.uint8) - # To RGB - a *= 255 - a = a.astype('uint8') - return a +def visualize(out_dirname, im, model): + + def create_subdir(layer_idx): + dirname = os.path.join(out_dirname, 'conv{}/'.format(layer_idx+1)) + dirname = os.path.dirname(dirname) + if not os.path.exists(dirname): + os.makedirs(dirname) + return dirname + + def save_to_ims(activations, dirname): + filename_len = len(str(len(activations))) + + # Save each feature map activation as its own image + for i, activation in enumerate(activations): + filename = '{num:0{width}}.png'.format(num=i, width=filename_len) + filename = os.path.join(dirname, filename) + save_im(filename, activation) + + # Save an image of all feature map activations in this layer + tiled_filename = os.path.join(out_dirname, 'conv{}.png'.format(layer_idx+1)) + tile_ims(tiled_filename, dirname) + + # Visualize each of the 5 convolutional layers in VGG + for layer_idx in range(5): + print('Visualizing activations for conv{}...'.format(layer_idx+1)) + activations = visualize_layer_activations(model, im.copy(), layer_idx) + dirname = create_subdir(layer_idx) + save_to_ims(activations, dirname) def main(args): @@ -71,8 +133,6 @@ def main(args): model_filename = args.model_filename image_filename = args.image_filename - print('Loading VGG model... ') - model = VGG() serializers.load_hdf5(model_filename, model) @@ -80,29 +140,9 @@ def main(args): cuda.get_device(gpu).use() model.to_gpu() - sample_im = read_im(image_filename) - - # Visualize each of the 5 convolutional layers in VGG - for layer_idx in range(5): - # Create the target directory if it doesn't already exist - dst_dir = os.path.join(out_dirname, 'layer_{}/'.format(layer_idx)) - dst_dir = os.path.dirname(dst_dir) - if not os.path.exists(dst_dir): - os.makedirs(dst_dir) - - print('Computing activations for layer {}...'.format(layer_idx)) - activations = get_activations(model, sample_im, layer_idx) - - filename_len = len(str(len(activations))) - for i, activation in enumerate(activations): - im = np.rollaxis(activation, 0, 3) # c, h, w -> h, w, c - filename = os.path.join(dst_dir, - '{num:0{width}}.jpg'.format(num=i, width=filename_len)) - - imgutil.save_im(filename, im) + im = read_im(image_filename) - tiled_filename = os.path.join(out_dirname, 'layer_{}.jpg'.format(layer_idx)) - imgutil.tile_ims(tiled_filename, dst_dir) + visualize(out_dirname, im, model) if __name__ == '__main__':