Raccoon Image Detection using Keras

1## Installation
2!pip install netron
3!pip install keras_sequential_ascii

 1
 2import tensorflow as tf
 3from tensorflow.keras.models import Sequential, load_model
 4from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
 5from tensorflow.keras.layers import Dense, Reshape, Flatten, Dropout, Activation, BatchNormalization
 6from tensorflow.keras.optimizers import SGD, Adam
 7from tensorflow.keras.callbacks import History 
 8from keras.callbacks.callbacks import ModelCheckpoint
 9
10
11print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
12tf.config.experimental.list_physical_devices('GPU')
13
14gpus = tf.config.experimental.list_physical_devices('GPU')
15if gpus:
16  # Restrict TensorFlow to only allocate 1GB of memory on the first GPU
17  try:
18    tf.config.experimental.set_virtual_device_configuration(
19        gpus[0],
20        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2048)])
21    logical_gpus = tf.config.experimental.list_logical_devices('GPU')
22    print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
23  except RuntimeError as e:
24    # Virtual devices must be set before GPUs have been initialized
25    print(e)

 1### Imports
 2
 3import numpy as np
 4import pandas as pd
 5
 6from keras import Model
 7from keras.applications.mobilenet import MobileNet, preprocess_input
 8from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, Callback
 9from keras.layers import Conv2D, Reshape
10from keras.utils import Sequence
11from keras.backend import epsilon
12import tensorflow as tf
13
14import os
15
16%matplotlib inline
17import matplotlib.pyplot as plt
18
19from PIL import Image
20from matplotlib.patches import Rectangle
21
22import csv
23
24from keras_sequential_ascii import keras2ascii

 1## Constants
 2
 3IMAGE_SIZE = 128
 4
 5CHANNELS = 3
 6
 7IMAGE_PATH = '/home/ashish/ai_ml/object_detection_dataset/raccoon_dataset-master/images'
 8
 9TRAIN_LABELS = 'train_labels.csv'
10
11PATIENCE = 10

1# Loading data
2train_df = pd.read_csv(TRAIN_LABELS)
3
4train_df.shape

1train_df.head()

1## Images 
2
3racoons = [IMAGE_PATH + "/" + f for f in os.listdir(IMAGE_PATH)]
4
5print('total raccoon images {}'.format(len(racoons)))

 1
 2limit = int((len(train_df.index)/25))
 3print(limit)
 4
 5for index in range(limit):
 6    filename = IMAGE_PATH + "/" + (train_df['filename'][index])
 7    width = train_df['width'][index]
 8    height = train_df['height'][index]
 9    xmin = train_df['xmin'][index]
10    ymin = train_df['ymin'][index]
11    xmax = train_df['xmax'][index]
12    ymax = train_df['ymax'][index]
13    imageObject = Image.open(filename)
14    print(filename)
15    print("W X H {}".format(imageObject.size))
16    #show Image object
17    plt.imshow(imageObject)
18    rect = Rectangle((xmin,ymin),xmax-xmin,ymax-ymin,linewidth=1,edgecolor='r',facecolor='none')
19    ax = plt.gca()
20    ax.add_patch(rect)
21    plt.show()    
22    

 1## Readinf csv file
 2
 3with open(TRAIN_LABELS) as csvFile:
 4    paths = []
 5    coords = np.zeros((sum(1 for line in csvFile) - 1, 4))    
 6#     print(coords)
 7    csvReader = csv.reader(csvFile, delimiter=",")
 8    print(csvReader)
 9    csvFile.seek(0)
10    next(csvReader, None)
11    for i, row in enumerate(csvReader):
12#         print('****************')
13#         print(row)
14        for j, r in enumerate(row):
15            if((j!= 0) & (j!=3)):
16                # Parse width, height, xmin, xmax, ymin, ymax to int and store in cordinates for each 
17                row[j] = int(r)
18                
19        path, width, height, class_, xmin, ymin, xmax, ymax = row
20        path = IMAGE_PATH +"/" + path
21        paths.append(path)
22        # Resize bounding box for training images as we have rescaled image to IMAGE_SIZE = 128
23        coords[i, 0] = xmin * IMAGE_SIZE / width
24        coords[i, 1] = ymin * IMAGE_SIZE / height
25        coords[i, 2] = xmax * IMAGE_SIZE / width
26        coords[i, 3] = ymax * IMAGE_SIZE / height
27        
28
29print(len(paths))
30print(coords.shape)

 1### Generate image batches 
 2# Create 3 dimensional array for each image holding 0.0 values
 3batch_images = np.zeros((len(paths), IMAGE_SIZE, IMAGE_SIZE, 3), dtype=np.float32)
 4# print(batch_images[1])
 5
 6for i, path in enumerate(paths):
 7    img = Image.open(path)
 8    img = img.resize((IMAGE_SIZE, IMAGE_SIZE))
 9    img = img.convert('RGB')
10    imgArray = np.array(img, dtype=np.float32)
11    imgArrayNormalised = preprocess_input(imgArray)
12    batch_images[i] = imgArrayNormalised
13    
14print(batch_images.shape)    

 1### Mobile net solution
 2
 3ALPHA = 1.0
 4input_shape = (IMAGE_SIZE, IMAGE_SIZE, CHANNELS)
 5model = MobileNet(input_shape = input_shape, include_top = False, alpha = ALPHA)
 6
 7# include_top = False => do not include classification layer
 8# alpha = 0..1 => Load Weight
 9
10# Freeze All layers
11for layer in model.layers:
12#     print(layer.name)
13    layer.trainable = False
14    
15keras2ascii(model)    

 1## Adding new Coordinate layer on top of existing layer
 2
 3# Add new convlution layer on top of existing model
 4print("Adding new layers now*********")
 5x = model.layers[-1].output
 6# Add new top layer which is a conv layer of the same size as the previous layer so that only 4 coords of BBox can be output
 7# In the line below kernel size should be 3 for img size 96, 4 for img size 128, 5 for img size 160 etc.
 8x = Conv2D(4, kernel_size=4, name='coords')(x)
 9# These are the 4 predicted coordinates of one BBox
10x = Reshape((4, ))(x)
11
12
13model = Model(inputs=model.input, outputs = x)
14keras2ascii(model)

 1### Define a custom loss function IoU which calculates Intersection Over Union
 2from keras import backend as K
 3
 4def loss(gt,pred):
 5    intersections = 0
 6    unions = 0
 7    diff_width = np.minimum(gt[:,0] + gt[:,2], pred[:,0] + pred[:,2]) - np.maximum(gt[:,0], pred[:,0])
 8    diff_height = np.minimum(gt[:,1] + gt[:,3], pred[:,1] + pred[:,3]) - np.maximum(gt[:,1], pred[:,1])
 9    intersection = diff_width * diff_height
10    
11    # Compute union
12    area_gt = gt[:,2] * gt[:,3]
13    area_pred = pred[:,2] * pred[:,3]
14    union = area_gt + area_pred - intersection
15
16#     Compute intersection and union over multiple boxes
17    for j, _ in enumerate(union):
18        if union[j] > 0 and intersection[j] > 0 and union[j] >= intersection[j]:
19            intersections += intersection[j]
20            unions += union[j]
21
22    # Compute IOU. Use epsilon to prevent division by zero
23    iou = np.round(intersections / (unions + epsilon()), 4)
24    iou = iou.astype(np.float32)
25    return iou
26
27
28def IoU(y_true, y_pred):
29    iou = tf.py_function(loss, [y_true, y_pred], tf.float32)
30    return iou

 1# Compile and configure model parameters
 2
 3coordinates = coords
 4
 5
 6
 7# Checkpoint best validation model
 8checkpoint = ModelCheckpoint("model-{IoU:.2f}.h5", verbose=1, save_best_only=True,
 9                              save_weights_only=True, mode="max", period=1) 
10# Stop early, if the validation error deteriorates
11stop = EarlyStopping(monitor="IoU", patience=PATIENCE, mode="max") 
12
13# Reduce learning rate if Validation IOU does not improve
14reduce_lr = ReduceLROnPlateau(monitor="IoU", factor=0.2, patience=10, min_lr=1e-7, verbose=1, mode="max")

1## Fit model
2
3model.compile(optimizer='Adam', loss='mse', metrics=[IoU])
4
5model.fit(batch_images, coordinates, verbose=1, epochs=64, batch_size=24, callbacks=[stop, checkpoint, reduce_lr])

 1## Pick and image and show box around face
 2imageNumber = 10
 3img_path = '/home/ashish/ai_ml/object_detection_dataset/raccoon_dataset-master/images/raccoon-{}.jpg'\
 4.format(imageNumber)
 5img_path = '/home/ashish/Downloads/raccoon_test_1.jpg'
 6imageObjectOriginal = Image.open(img_path)
 7imageObjectScaled = Image.open(img_path)
 8imageObjectScaled = imageObjectScaled.resize((IMAGE_SIZE, IMAGE_SIZE))
 9
10## Extract fearures from images for model to understand
11features_scaled = preprocess_input(np.array(imageObjectScaled, dtype=np.float32))
12print(features_scaled.shape)
13region = model.predict(x=np.array([features_scaled]))[0]
14
15print(region)

 1## Draw bounding boxes for scaled and unscaled images
 2
 3print("Box on scaled image")
 4print('Scaled width & height ({}, {})'.format(imageObjectScaled.width, imageObjectScaled.height))
 5print("({}, {}) , ({}, {})".format(xmin, ymin, xmax, ymax))
 6## Show box in scaled image
 7plt.imshow(imageObjectScaled)
 8width = IMAGE_SIZE
 9height = IMAGE_SIZE
10xmin = region[0]
11ymin = region[1]
12xmax = region[2]
13ymax = region[3]
14rect = Rectangle((xmin,ymin),xmax-xmin,ymax-ymin,linewidth=1,color='cyan',facecolor='none', alpha=0.3)
15ax = plt.gca()
16ax.add_patch(rect)
17plt.show()  
18
19print("Box on unscaled image")
20## Show box on unscaled image
21width, height = imageObjectOriginal.size
22print('Original width & height ({}, {})'.format(width, height))
23xmin = (region[0] * width/ IMAGE_SIZE)
24ymin = (region[1] * height/ IMAGE_SIZE)
25xmax = (region[2] * width / IMAGE_SIZE)
26ymax = (region[3] * height /IMAGE_SIZE)
27
28print("({}, {}) , ({}, {})".format(xmin, ymin, xmax, ymax))
29plt.imshow(imageObjectOriginal)
30rect = Rectangle((xmin,ymin),xmax-xmin,ymax-ymin,linewidth=1,color='cyan',facecolor='none', alpha=0.3)
31ax = plt.gca()
32ax.add_patch(rect)
33plt.show()