From ff1a17f52f363a3901f2b7a2d720b7745e1c3bc4 Mon Sep 17 00:00:00 2001 From: Alex Parinov Date: Thu, 10 Oct 2019 22:10:53 +0300 Subject: [PATCH] Benchmark improvements (#413) * Fix benchmarking issues * Use copy=True when converting Pillow Image to np.array * Update benchmarking results --- README.md | 32 ++++++++++----------- benchmark/benchmark.py | 64 +++++++++++++++++++++++++++--------------- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 6d2e2f58c..33b7806c1 100644 --- a/README.md +++ b/README.md @@ -205,22 +205,22 @@ The table shows how many images per second can be processed on a single core, hi | | albumentations
0.4.0 | imgaug
0.2.9 | torchvision (Pillow backend)
0.4.0 | torchvision (Pillow-SIMD backend)
0.4.0 | keras
2.3.1 | augmentor
0.2.6 | solt
0.1.8 | |------------------------|:--------------:|:------:|:--------------------:|:-------------------------:|:-----:|:---------:|:-----:| -| HorizontalFlip | 1393 | 880 | **3799** | 3670 | 543 | 3744 | 190 | -| VerticalFlip | **3908** | 2120 | 2856 | 3294 | 3901 | 2752 | 3178 | -| Rotate | **296** | 251 | 82 | 104 | 10 | 37 | 74 | -| ShiftScaleRotate | **640** | 428 | 78 | 109 | 13 | - | - | -| Brightness | **1723** | 1010 | 303 | 373 | 132 | 292 | 1535 | -| Contrast | **1753** | 1126 | 215 | 261 | - | 209 | 1535 | -| BrightnessContrast | **1698** | 566 | 125 | 152 | - | 118 | 794 | -| ShiftRGB | **1698** | 1022 | - | - | 507 | - | - | -| ShiftHSV | **341** | 339 | 34 | 45 | - | - | 106 | -| Gamma | **1809** | - | 725 | 775 | - | - | 571 | -| Grayscale | 3564 | 225 | 662 | 800 | - | 2042 |**4723**| -| RandomCrop64 | **296061** | 2354 | 25418 | 49619 | - | 19515 | 38583 | -| PadToSize512 | **2690** | - | 396 | 433 | - | - | 2381 | -| Resize512 | 573 | 413 | 247 | **1009** | - | 232 | 552 | -| RandomSizedCrop_64_512 | **1865** | 782 | 421 | 1714 | - | 409 | 1751 | -| Equalize | **535** | 463 | - | - | - | 298 | - | +| HorizontalFlip | 961 | 754 | **1246** | 1251 | 669 | 1154 | 619 | +| VerticalFlip | **3941** | 2069 | 1105 | 1150 | 3884 | 1054 | 3540 | +| Rotate | **375** | 300 | 83 | 120 | 18 | 36 | 91 | +| ShiftScaleRotate | **664** | 454 | 75 | 116 | 23 | - | - | +| Brightness | **1806** | 1067 | 260 | 320 | 133 | 252 | 1694 | +| Contrast | **1701** | 1123 | 190 | 241 | - | 184 | 1699 | +| BrightnessContrast | **1749** | 577 | 114 | 143 | - | 112 | 880 | +| ShiftRGB | **1813** | 984 | - | - | 509 | - | - | +| ShiftHSV | **349** | 340 | 35 | 45 | - | - | 106 | +| Gamma | **1926** | - | 549 | 580 | - | - | 701 | +| Grayscale | **3688** | 307 | 487 | 574 | - | 872 | 2927 | +| RandomCrop64 | **602010** | 2908 | 22398 | 33850 | - | 14267 | 38450 | +| PadToSize512 | **2749** | - | 350 | 378 | - | - | 2370 | +| Resize512 | 576 | 427 | 211 | **648** | - | 213 | 568 | +| RandomSizedCrop_64_512 | **2223** | 715 | 334 | 1023 | - | 339 | 1949 | +| Equalize | **466** | 460 | - | - | - | 256 | - | Python and library versions: Python 3.7.3, numpy 1.17.2, pillow 6.2.0, pillow-simd 6.0.0.post0, opencv-python 4.1.1.26, scikit-image 0.15.0, scipy 1.3.0. diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 2d2d9b4f5..eec0cb5f4 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -98,13 +98,18 @@ def imgaug(self, img): return self.imgaug_transform.augment_image(img) def augmentor(self, img): - return self.augmentor_op.perform_operation([img]) + img = self.augmentor_op.perform_operation([img])[0] + return np.array(img, np.uint8, copy=True) def solt(self, img): dc = sld.DataContainer(img, "I") dc = self.solt_stream(dc) return dc.data[0] + def torchvision(self, img): + img = self.torchvision_transform(img) + return np.array(img, np.uint8, copy=True) + def is_supported_by(self, library): if library == "imgaug": return hasattr(self, "imgaug_transform") @@ -112,6 +117,8 @@ def is_supported_by(self, library): return hasattr(self, "augmentor_op") or hasattr(self, "augmentor_pipeline") elif library == "solt": return hasattr(self, "solt_stream") + elif library == "torchvision": + return hasattr(self, "torchvision_transform") else: return hasattr(self, library) @@ -133,7 +140,7 @@ def albumentations(self, img): else: return albumentations.hflip(img) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.hflip(img) def keras(self, img): @@ -152,7 +159,7 @@ def __init__(self): def albumentations(self, img): return albumentations.vflip(img) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.vflip(img) def keras(self, img): @@ -171,7 +178,7 @@ def __init__(self): def albumentations(self, img): return albumentations.rotate(img, angle=-45) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.rotate(img, angle=-45, resample=Image.BILINEAR) def keras(self, img): @@ -187,7 +194,7 @@ def __init__(self): def albumentations(self, img): return albumentations.brightness_contrast_adjust(img, beta=0.5, beta_by_max=True) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.adjust_brightness(img, brightness_factor=1.5) def keras(self, img): @@ -203,7 +210,7 @@ def __init__(self): def albumentations(self, img): return albumentations.brightness_contrast_adjust(img, alpha=1.5) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.adjust_contrast(img, contrast_factor=1.5) @@ -227,7 +234,7 @@ def __init__(self): def albumentations(self, img): return albumentations.brightness_contrast_adjust(img, alpha=1.5, beta=0.5, beta_by_max=True) - def torchvision(self, img): + def torchvision_transform(self, img): img = torchvision.adjust_brightness(img, brightness_factor=1.5) img = torchvision.adjust_contrast(img, contrast_factor=1.5) return img @@ -235,7 +242,7 @@ def torchvision(self, img): def augmentor(self, img): for operation in self.augmentor_pipeline.operations: img, = operation.perform_operation([img]) - return img + return np.array(img, np.uint8, copy=True) class ShiftScaleRotate(BenchmarkTest): @@ -247,7 +254,7 @@ def __init__(self): def albumentations(self, img): return albumentations.shift_scale_rotate(img, angle=-45, scale=2, dx=0.2, dy=0.2) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.affine(img, angle=45, translate=(50, 50), scale=2, shear=0, resample=Image.BILINEAR) def keras(self, img): @@ -262,7 +269,7 @@ def __init__(self): def albumentations(self, img): return albumentations.shift_hsv(img, hue_shift=20, sat_shift=20, val_shift=20) - def torchvision(self, img): + def torchvision_transform(self, img): img = torchvision.adjust_hue(img, hue_factor=0.1) img = torchvision.adjust_saturation(img, saturation_factor=1.2) img = torchvision.adjust_brightness(img, brightness_factor=1.2) @@ -301,17 +308,17 @@ def __init__(self): def albumentations(self, img): return albumentations.random_crop(img, crop_height=64, crop_width=64, h_start=0, w_start=0) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.crop(img, i=0, j=0, h=64, w=64) class RandomSizedCrop_64_512(BenchmarkTest): def __init__(self): - - self.augmentor_op = [ - Operations.Crop(probability=1, width=64, height=64, centre=False), - Operations.Resize(probability=1, width=512, height=512, resample_filter="BILINEAR"), - ] + self.augmentor_pipeline = Pipeline() + self.augmentor_pipeline.add_operation(Operations.Crop(probability=1, width=64, height=64, centre=False)) + self.augmentor_pipeline.add_operation( + Operations.Resize(probability=1, width=512, height=512, resample_filter="BILINEAR") + ) self.imgaug_transform = iaa.Sequential( [iaa.CropToFixedSize(width=64, height=64), iaa.Scale(size=512, interpolation="linear")] ) @@ -324,10 +331,11 @@ def albumentations(self, img): return albumentations.resize(img, height=512, width=512) def augmentor(self, img): - img = self.augmentor_op[0].perform_operation([img])[0] - return self.augmentor_op[1].perform_operation([img]) + for operation in self.augmentor_pipeline.operations: + img, = operation.perform_operation([img]) + return np.array(img, np.uint8, copy=True) - def torchvision(self, img): + def torchvision_transform(self, img): img = torchvision.crop(img, i=0, j=0, h=64, w=64) return torchvision.resize(img, (512, 512)) @@ -350,7 +358,7 @@ def __init__(self): def albumentations(self, img): return albumentations.pad(img, min_height=512, min_width=512) - def torchvision(self, img): + def torchvision_transform(self, img): if img.size[0] < 512: img = torchvision.pad(img, (int((1 + 512 - img.size[0]) / 2), 0), padding_mode="reflect") if img.size[1] < 512: @@ -367,7 +375,7 @@ def __init__(self): def albumentations(self, img): return albumentations.resize(img, height=512, width=512) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.resize(img, (512, 512)) @@ -378,7 +386,7 @@ def __init__(self): def albumentations(self, img): return albumentations.gamma_transform(img, gamma=0.5) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.adjust_gamma(img, gamma=0.5) @@ -391,9 +399,19 @@ def __init__(self): def albumentations(self, img): return albumentations.to_gray(img) - def torchvision(self, img): + def torchvision_transform(self, img): return torchvision.to_grayscale(img, num_output_channels=3) + def solt(self, img): + dc = sld.DataContainer(img, "I") + dc = self.solt_stream(dc) + return cv2.cvtColor(dc.data[0], cv2.COLOR_GRAY2RGB) + + def augmentor(self, img): + img = self.augmentor_op.perform_operation([img])[0] + img = np.array(img, np.uint8, copy=True) + return np.dstack([img, img, img]) + class Posterize(BenchmarkTest): def albumentations(self, img):