Object detection: JPEG compression¶

Note: Object detection requirements.

Warning: Runtimes can be several hours even on clusters.

We compared the performance of models from FaceBook’s Detectron project and YOLOv3 model from Joseph Redmon, when different error sources were added. The models from FaceBook’s Detectron project were FasterRCNN, MaskRCNN and RetinaNet.

[1]:

import re
from abc import ABC, abstractmethod

import matplotlib.pyplot as plt
from PIL import Image

from dpemu import runner
from dpemu.dataset_utils import load_coco_val_2017
from dpemu.filters.image import JPEG_Compression
from dpemu.ml_utils import run_ml_module_using_cli, load_yolov3
from dpemu.nodes import Array, Series
from dpemu.plotting_utils import print_results_by_model, visualize_scores
from dpemu.utils import get_project_root

We used 118 287 jpg images (COCO train2017) as the train set and 5000 images (COCO val2017) as the test set to calculate the mAP-50 scores.

[ ]:

def get_data():
    imgs, _, _, img_filenames = load_coco_val_2017()
    return imgs, img_filenames

[2]:

def get_err_root_node():
    err_node = Array()
    err_root_node = Series(err_node)
    err_node.addfilter(JPEG_Compression("quality"))
    return err_root_node

Examples from run_yolo_example.py¶

quality: 5¶

quality: 15¶

quality: 25¶

quality: 40¶

[3]:

def get_err_params_list():
    return [{"quality": q} for q in range(5, 45, 5)]

[4]:

class Preprocessor:

    def run(self, _, imgs, params):
        img_filenames = params["img_filenames"]

        for i, img_arr in enumerate(imgs):
            img = Image.fromarray(img_arr)
            path_to_img = f"{get_project_root()}/tmp/val2017/" + img_filenames[i]
            img.save(path_to_img, "jpeg", quality=100)

        return None, imgs, {}

Detectron’s model zoo had pretrained weights for FasterRCNN, MaskRCNN and RetinaNet. YOLOv3’s weights were trained by us, using the Kale cluster of University of Helsinki. The training took approximately five days when two NVIDIA Tesla V100 GPUs were used.

[5]:

class YOLOv3Model:

    def run(self, _, imgs, params):
        path_to_yolov3_weights, path_to_yolov3_cfg = load_yolov3()

        cline = f"{get_project_root()}/libs/darknet/darknet detector map {get_project_root()}/data/coco.data \
            {path_to_yolov3_cfg} {path_to_yolov3_weights}"
        out = run_ml_module_using_cli(cline, show_stdout=False)

        match = re.search(r"\(mAP@0.50\) = (\d+\.\d+)", out)
        return {"mAP-50": round(float(match.group(1)), 3)}


class AbstractDetectronModel(ABC):

    def run(self, _, imgs, params):
        path_to_cfg = self.get_path_to_cfg()
        url_to_weights = self.get_url_to_weights()

        cline = f"""{get_project_root()}/libs/Detectron/tools/test_net.py \
            --cfg {path_to_cfg} \
            TEST.WEIGHTS {url_to_weights} \
            NUM_GPUS 1 \
            TEST.DATASETS '("coco_2017_val",)' \
            MODEL.MASK_ON False \
            OUTPUT_DIR {get_project_root()}/tmp \
            DOWNLOAD_CACHE {get_project_root()}/tmp"""
        out = run_ml_module_using_cli(cline, show_stdout=False)

        match = re.search(r"IoU=0.50      \| area=   all \| maxDets=100 ] = (\d+\.\d+)", out)
        return {"mAP-50": round(float(match.group(1)), 3)}

    @abstractmethod
    def get_path_to_cfg(self):
        pass

    @abstractmethod
    def get_url_to_weights(self):
        pass


class FasterRCNNModel(AbstractDetectronModel):

    def get_path_to_cfg(self):
        return f"{get_project_root()}/libs/Detectron/configs/12_2017_baselines/e2e_faster_rcnn_X-101-64x4d-FPN_1x.yaml"

    def get_url_to_weights(self):
        return (
            "https://dl.fbaipublicfiles.com/detectron/35858015/12_2017_baselines/"
            "e2e_faster_rcnn_X-101-64x4d-FPN_1x.yaml.01_40_54.1xc565DE/output/train/"
            "coco_2014_train%3Acoco_2014_valminusminival/generalized_rcnn/model_final.pkl"
        )


class MaskRCNNModel(AbstractDetectronModel):

    def get_path_to_cfg(self):
        return f"{get_project_root()}/libs/Detectron/configs/12_2017_baselines/e2e_mask_rcnn_X-101-64x4d-FPN_1x.yaml"

    def get_url_to_weights(self):
        return (
            "https://dl.fbaipublicfiles.com/detectron/36494496/12_2017_baselines/"
            "e2e_mask_rcnn_X-101-64x4d-FPN_1x.yaml.07_50_11.fkwVtEvg/output/train/"
            "coco_2014_train%3Acoco_2014_valminusminival/generalized_rcnn/model_final.pkl"
        )


class RetinaNetModel(AbstractDetectronModel):

    def get_path_to_cfg(self):
        return f"{get_project_root()}/libs/Detectron/configs/12_2017_baselines/retinanet_X-101-64x4d-FPN_1x.yaml"

    def get_url_to_weights(self):
        return (
            "https://dl.fbaipublicfiles.com/detectron/36768875/12_2017_baselines/"
            "retinanet_X-101-64x4d-FPN_1x.yaml.08_34_37.FSXgMpzP/output/train/"
            "coco_2014_train%3Acoco_2014_valminusminival/retinanet/model_final.pkl"
        )

[6]:

def get_model_params_dict_list():
    return [
        {"model": FasterRCNNModel, "params_list": [{}]},
        {"model": MaskRCNNModel, "params_list": [{}]},
        {"model": RetinaNetModel, "params_list": [{}]},
        {"model": YOLOv3Model, "params_list": [{}]},
    ]

[7]:

def visualize(df):
    visualize_scores(
        df,
        score_names=["mAP-50"],
        is_higher_score_better=[True],
        err_param_name="quality",
        title="Object detection with JPEG compression"
    )
    plt.show()

[8]:

def main():
    imgs, img_filenames = get_data()

    df = runner.run(
        train_data=None,
        test_data=imgs,
        preproc=Preprocessor,
        preproc_params={"img_filenames": img_filenames},
        err_root_node=get_err_root_node(),
        err_params_list=get_err_params_list(),
        model_params_dict_list=get_model_params_dict_list(),
        n_processes=1
    )

    print_results_by_model(df)
    visualize(df)

[9]:

main()

loading annotations into memory...
Done (t=0.54s)
creating index...
index created!

  0%|          | 0/8 [00:00<?, ?it/s]

 12%|█▎        | 1/8 [48:22<5:38:35, 2902.14s/it]

 25%|██▌       | 2/8 [1:37:02<4:50:44, 2907.49s/it]

 38%|███▊      | 3/8 [2:26:01<4:03:06, 2917.21s/it]

 50%|█████     | 4/8 [3:15:03<3:14:58, 2924.60s/it]

 62%|██████▎   | 5/8 [4:04:14<2:26:37, 2932.38s/it]

 75%|███████▌  | 6/8 [4:53:27<1:37:56, 2938.49s/it]

 88%|████████▊ | 7/8 [5:42:35<49:01, 2941.60s/it]

100%|██████████| 8/8 [6:31:51<00:00, 2945.84s/it]

FasterRCNN #1

	mAP-50	quality	time_err	time_pre	time_mod
0	0.092	5	64.709	122.873	808.970
1	0.277	10	66.442	127.487	809.532
2	0.407	15	68.678	134.865	811.471
3	0.479	20	70.196	137.164	813.800
4	0.523	25	70.890	139.103	814.348
5	0.550	30	72.191	142.072	812.131
6	0.566	35	73.478	144.595	812.938
7	0.576	40	74.204	145.890	813.014

MaskRCNN #1

	mAP-50	quality	time_err	time_pre	time_mod
0	0.105	5	64.709	122.873	796.518
1	0.301	10	66.442	127.487	809.272
2	0.431	15	68.678	134.865	810.367
3	0.498	20	70.196	137.164	808.612
4	0.539	25	70.890	139.103	811.646
5	0.563	30	72.191	142.072	811.534
6	0.576	35	73.478	144.595	810.514
7	0.587	40	74.204	145.890	812.432

RetinaNet #1

	mAP-50	quality	time_err	time_pre	time_mod
0	0.122	5	64.709	122.873	1007.092
1	0.305	10	66.442	127.487	1009.791
2	0.412	15	68.678	134.865	1014.549
3	0.466	20	70.196	137.164	1011.720
4	0.500	25	70.890	139.103	1015.210
5	0.521	30	72.191	142.072	1014.208
6	0.535	35	73.478	144.595	1009.012
7	0.543	40	74.204	145.890	1010.712

YOLOv3 #1

	mAP-50	quality	time_err	time_pre	time_mod
0	0.103	5	64.709	122.873	98.066
1	0.312	10	66.442	127.487	92.068
2	0.421	15	68.678	134.865	92.472
3	0.466	20	70.196	137.164	92.607
4	0.493	25	70.890	139.103	92.315
5	0.508	30	72.191	142.072	92.577
6	0.515	35	73.478	144.595	92.671
7	0.522	40	74.204	145.890	92.401

../_images/case_studies_Object_Detection_JPEG_Compression_20_26.png

The notebook for this case study can be found here.