in Machine Learning

Comparing Cityscapes, Mapillary and BDD for Segmentation Datasets

Performing exploratory data analysis on the three most widely used semantic segmentation datasets in the Autonomous Vehicles domain.

import cv2
from collections import namedtuple, Counter
from glob import glob
import matplotlib.pyplot as plt
import numpy as np
import os
import pickle
from skimage.io import imread
from tqdm.notebook import tqdm

from bdd_config import labels as bdd_labels
from cityscapes_config import labels as cityscapes_labels
from mapillary_config import labels as mapillary_labels
from mapillary_config import name2cat as mapillary_name2cat
cityscapes = {
    'train':{
        'images':sorted(glob('cityscapes/leftImg8bit/train/*/*.png')),
        'masks':sorted(glob('cityscapes/gtFine/train/*/*gtFine_labelIds.png'))
    },
    'val':{
        'images':sorted(glob('cityscapes/leftImg8bit/val/*/*.png')),
        'masks':sorted(glob('cityscapes/gtFine/val/*/*gtFine_labelIds.png'))
    },
    'mappings':{}
}

mapillary = {
    'train':{
        'images':sorted(glob('mapillary-vistas/mapillary-vistas-dataset_public_v1.1/train/images/*')),
        'masks':sorted(glob('mapillary-vistas/mapillary-vistas-dataset_public_v1.1/train/masks/*'))
    },
    'val':{
        'images':sorted(glob('mapillary-vistas/mapillary-vistas-dataset_public_v1.1/val/images/*')),
        'masks':sorted(glob('mapillary-vistas/mapillary-vistas-dataset_public_v1.1/val/masks/*'))
    },
    'mappings':{}
}

bdd = {
    'train':{
        'images':sorted(glob('BDD_SEGM/train/images/*')),
        'masks':sorted(glob('BDD_SEGM/train/masks/*'))
    },
    'val':{
        'images':sorted(glob('BDD_SEGM/val/images/*')),
        'masks':sorted(glob('BDD_SEGM/val/masks/*'))
    },
    'mappings':{}
}

Comparing the number of images in train and validation split

plt.bar([0, 0.25], 
        [len(cityscapes['train']['images']),
         len(cityscapes['val']['images'])],
        width=0.20,
        color=[(.1, .46, .82),
               (.1, .46, .82)])
plt.bar([.8, 1.05], 
        [len(mapillary['train']['images']),
         len(mapillary['val']['images'])],
        width=0.20,
        color=[(.23, 0.3, .51),
               (.23, 0.3, .51)])
plt.bar([1.6, 1.85], 
        [len(bdd['train']['images']),
         len(bdd['val']['images'])],
        width=0.20,
        color=[(.93, 0.32, .31),
               (.93, 0.32, .31)])
x_cords = [0, 0.25, .8, 1.05, 1.6, 1.85]
plt.xticks(x_cords, ['train', 'val'] * 3, rotation=45)
plt.ylabel('images')
plt.legend(['cityscapes', 'mapillary', 'bdd'])
plt.show()

Visualizing random samples

    color_map = dataset['mappings']['id2color']
    r = c = int(count ** 0.5)
    plt.subplots(r, c, figsize=(16, 8))
    indices = np.random.randint(0, high=len(dataset['train']['images']), size=count)
    for i, idx in enumerate(indices):
        plt.subplot(r, c, i+1)
        plt.title(title)
        image = imread(dataset['train']['images'][idx])
        mask = imread(dataset['train']['masks'][idx])
        alpha = 0.45
        color_mask = image.copy()
        for id in np.unique(mask):
            color_mask[mask==id] = color_map[id]
        image_overlay = cv2.addWeighted(image, alpha, color_mask, 1-alpha, 0)
        plt.imshow(image_overlay)
    plt.show()
    
display_annotations(cityscapes, 4, 'Cityscapes')
display_annotations(mapillary, 4, 'Mapillary')
display_annotations(bdd, 4, 'BDD')    

Mappings for convenience

cityscapes['mappings']['id2name'] = {
    label.id: label.name for label in cityscapes_labels}
cityscapes['mappings']['id2color'] = {
    label.id: label.color for label in cityscapes_labels}
cityscapes['mappings']['name2color'] = {label.name: np.array(
    label.color) / 255. for label in cityscapes_labels}
cityscapes['mappings']['trainable_ids'] = [
    label.id for label in cityscapes_labels if not label.ignoreInEval]
cityscapes['mappings']['name2cat'] = {
    label.name: label.category for label in cityscapes_labels}


mapillary['mappings']['name2color'] = {}
for label in mapillary_labels:
    mapillary['mappings']['name2color'][label['readable']
                                        ] = np.array(label['color']) / 255.
mapillary_label_list = sorted(mapillary['mappings']['name2color'].keys())
mapillary_name2id = {k: v for v, k in enumerate(mapillary_label_list)}
mapillary['mappings']['id2name'] = {v: k for k, v in mapillary_name2id.items()}
mapillary['mappings']['id2color'] = {id: np.uint8(
    mapillary['mappings']['name2color'][cls_name] * 255) for cls_name, id in mapillary_name2id.items()}
mapillary['mappings']['trainable_ids'] = None
mapillary['mappings']['name2cat'] = mapillary_name2cat

bdd['mappings']['id2name'] = {label.id: label.name for label in bdd_labels}
bdd['mappings']['id2color'] = {label.id: label.color for label in bdd_labels}
bdd['mappings']['name2color'] = {label.name: np.array(
    label.color) / 255. for label in bdd_labels}
bdd['mappings']['trainable_ids'] = [
    label.id for label in bdd_labels if not label.ignoreInEval]
bdd['mappings']['name2cat'] = {
    label.name: label.category for label in bdd_labels}
def comput_statistics(dataset):
    id2name = dataset['mappings']['id2name']
    trainable_ids = dataset['mappings']['trainable_ids']
    class_distribution = Counter()
    class_occurance = Counter()
    if not trainable_ids:
        trainable_ids = list(id2name.keys())
    for mask_path in tqdm(dataset['train']['masks'] + dataset['val']['masks']):
        mask = imread(mask_path, as_gray=True)
        ids, counts = np.unique(mask, return_counts=True)
        count_dict = {id2name[id]: count for id,
                      count in zip(ids, counts) if id in trainable_ids}
        occurance_dict = {id2name[id]: 1 for id in ids if id in trainable_ids}
        class_distribution.update(count_dict)
        class_occurance.update(occurance_dict)
    return class_distribution, class_occurance


def plot_class_distribution(dataset,
                            class_distribution,
                            h=6,
                            w=24):
    name2color = dataset['mappings']['name2color']
    plt.figure(figsize=(w, h))
    classes = list(class_distribution.keys())
    counts = list(class_distribution.values())
    normalized_counts = np.array(counts) * 100 / np.sum(counts)
    plt.barh(np.arange(len(classes)),
             normalized_counts,
             height=0.5,
             color=[name2color[cls] for cls in classes])
    plt.yticks(np.arange(len(classes)),
               classes,
               rotation='horizontal')
    plt.xlabel('normalized pixel count')
    plt.ylabel('classes')
    
def plot_class_category_distribution(dataset, 
                                     class_distribution,
                                     category):
    name2cat = dataset['mappings']['name2cat']
    name2color = dataset['mappings']['name2color']
    category_class_distribution = \
        {label: class_distribution[label]
         for label, cat in name2cat.items()
         if  cat == category and class_distribution[label] != 0}
    classes = list(category_class_distribution.keys())
    counts = list(category_class_distribution.values())
    normalized_counts = np.array(counts)*100/np.sum(counts)
    plt.bar(np.arange(len(classes)),
             normalized_counts,
             width=0.5,
             color=[name2color[cls] for cls in classes])
    plt.xticks(np.arange(len(classes)),
               classes,
               rotation=45)
    plt.ylabel('normalized pixel count')
    plt.xlabel('classes')
    
    
def plot_category_distribution(dataset, class_distribution, color):
    name2cat = dataset['mappings']['name2cat']
    category_distribution = Counter()

    for cls_, count_ in class_distribution.items():
        category_distribution.update({
            name2cat[cls_]:count_
        })
    categories = ['construction',
                  'flat',
                  'human',
                  'nature',
                  'object',
                  'sky',
                  'vehicle']
    counts = [category_distribution[cat] for cat in categories]
    normalized_counts = np.array(counts)*100/np.sum(counts)
    plt.bar(np.arange(len(categories)),
             normalized_counts,
             width=0.5, color=[color]*len(categories))
    plt.xticks(np.arange(len(categories)),
               categories,
               rotation=45)
    plt.ylabel('normalized pixel count')
    plt.xlabel('categories')
    
    
def plot_class_occurance(dataset, class_occurance, h=6, w=24):
    name2color= dataset['mappings']['name2color']
    plt.figure(figsize=(w, h))
    classes = list(class_occurance.keys())
    counts = list(class_occurance.values())
    plt.barh(np.arange(len(classes)),
             counts,
             height=0.5,
             color=[name2color[cls] for cls in classes])
    plt.yticks(np.arange(len(classes)),
               classes,
               rotation='horizontal')
    plt.xlabel('images')
    plt.ylabel('classes')
def dump_dicts(fname, class_distribution, class_occurance):
    with open(fname+'_cls_dist.pkl', 'wb') as f:
        pickle.dump(class_distribution, f)
    with open(fname+'_cls_occr.pkl', 'wb') as f:
        pickle.dump(class_occurance, f)
cityscapes_class_distribution, cityscapes_class_occurance = comput_statistics(cityscapes,
                                                                              cityscapes_id2name,
                                                                              cityscapes_trainable_ids)
mapillary_class_distribution, mapillary_class_occurance = comput_statistics(mapillary,
                                                                            mapillary_id2name)
bdd_class_distribution, bdd_class_occurance = comput_statistics(bdd,
                                                                bdd_id2name,
                                                                bdd_trainable_ids)
dump_dicts('cityscapes', cityscapes_class_distribution, cityscapes_class_occurance)
dump_dicts('mapillary', mapillary_class_distribution, mapillary_class_occurance)
dump_dicts('bdd', bdd_class_distribution, bdd_class_occurance)

Visualizing the class level distribution

plot_class_distribution(cityscapes,
                        cityscapes_class_distribution,
                        h=10,
                        w=32)
plt.title('Cityscapes')
plot_class_distribution(mapillary,
                        mapillary_class_distribution,
                        h=14,
                        w=32)
plt.title('Mapillary')
plot_class_distribution(bdd,
                        bdd_class_distribution, 
                        h=10,
                        w=32)
plt.title('Berkeley deep drive')

Text(0.5, 1.0, ‘Berkeley deep drive’)

Classes present in ‘construction’ category

plt.subplots(1, 3, figsize=(20, 8))
cat = 'construction'

plt.subplot(1, 3, 1)
plt.title('Cityscapes')
plot_class_category_distribution(cityscapes, cityscapes_class_distribution, cat)

plt.subplot(1, 3, 2)
plt.title('Mapillary')
plot_class_category_distribution(mapillary, mapillary_class_distribution, cat)

plt.subplot(1, 3, 3)
plt.title('Berkeley deep drive')
plot_class_category_distribution(bdd, bdd_class_distribution, cat)

Classes present in ‘flat’ category

cat = 'flat'

plt.subplot(1, 3, 1)
plt.title('Cityscapes')
plot_class_category_distribution(cityscapes, cityscapes_class_distribution, cat)

plt.subplot(1, 3, 2)
plt.title('Mapillary')
plot_class_category_distribution(mapillary, mapillary_class_distribution, cat)

plt.subplot(1, 3, 3)
plt.title('Berkeley deep drive')
plot_class_category_distribution(bdd, bdd_class_distribution, cat)

Classes present in ‘human’ category

plt.subplots(1, 3, figsize=(20, 8))
cat = 'human'

plt.subplot(1, 3, 1)
plt.title('Cityscapes')
plot_class_category_distribution(cityscapes, cityscapes_class_distribution, cat)

plt.subplot(1, 3, 2)
plt.title('Mapillary')
plot_class_category_distribution(mapillary, mapillary_class_distribution, cat)

plt.subplot(1, 3, 3)
plt.title('Berkeley deep drive')
plot_class_category_distribution(bdd, bdd_class_distribution, cat)

Classes present in ‘nature’ category

plt.subplots(1, 3, figsize=(20, 8))
cat = 'nature'

plt.subplot(1, 3, 1)
plt.title('Cityscapes')
plot_class_category_distribution(cityscapes, cityscapes_class_distribution, cat)

plt.subplot(1, 3, 2)
plt.title('Mapillary')
plot_class_category_distribution(mapillary, mapillary_class_distribution, cat)

plt.subplot(1, 3, 3)
plt.title('Berkeley deep drive')
plot_class_category_distribution(bdd, bdd_class_distribution, cat)

 

Classes present in ‘object’ category

plt.subplots(1, 3, figsize=(20, 8))
cat = 'object'

plt.subplot(1, 3, 1)
plt.title('Cityscapes')
plot_class_category_distribution(cityscapes, cityscapes_class_distribution, cat)

plt.subplot(1, 3, 2)
plt.title('Mapillary')
plot_class_category_distribution(mapillary, mapillary_class_distribution, cat)

plt.subplot(1, 3, 3)
plt.title('Berkeley deep drive')
plot_class_category_distribution(bdd, bdd_class_distribution, cat)

Classes present in ‘vehicle’ category

plt.subplots(1, 3, figsize=(20, 8))
cat = 'vehicle'

plt.subplot(1, 3, 1)
plt.title('Cityscapes')
plot_class_category_distribution(cityscapes, cityscapes_class_distribution, cat)

plt.subplot(1, 3, 2)
plt.title('Mapillary')
plot_class_category_distribution(mapillary, mapillary_class_distribution, cat)

plt.subplot(1, 3, 3)
plt.title('Berkeley deep drive')
plot_class_category_distribution(bdd, bdd_class_distribution, cat)

Visualizing the category level distribution

plt.subplots(1, 3, figsize=(20, 8))

plt.subplot(1, 3, 1)
plt.title('Cityscapes')
plot_category_distribution(cityscapes, cityscapes_class_distribution, (.1, .46, .82))

plt.subplot(1, 3, 2)
plt.title('Mapillary')
plot_category_distribution(mapillary, mapillary_class_distribution, (.23, 0.3, .51))

plt.subplot(1, 3, 3)
plt.title('Berkeley deep drive')
plot_category_distribution(bdd, bdd_class_distribution, (.93, 0.32, .31))

Visualizing number images a given class is present in

plot_class_occurance(cityscapes, cityscapes_class_occurance)
plt.title('Cityscapes')
plot_class_occurance(mapillary, mapillary_class_occurance, 12, 24)
plt.title('Mapillary')
plot_class_occurance(bdd, bdd_class_occurance)
plt.title('Berkeley deep drive')
Text(0.5, 1.0, 'Berkeley deep drive')