diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py
index 4547398a0..89b80aff2 100644
--- a/maskrcnn_benchmark/structures/segmentation_mask.py
+++ b/maskrcnn_benchmark/structures/segmentation_mask.py
@@ -48,31 +48,53 @@ def __init__(self, masks, size):
             initializing source data intact.
         """
 
+        assert isinstance(size, (list, tuple))
+        assert len(size) == 2
+
         if isinstance(masks, torch.Tensor):
             # The raw data representation is passed as argument
             masks = masks.clone()
         elif isinstance(masks, (list, tuple)):
             if len(masks) == 0:
-                masks = torch.empty([0, size[1], size[0]]) # num_instances = 0!
+                masks = torch.empty([0, size[1], size[0]])  # num_instances = 0!
             elif isinstance(masks[0], torch.Tensor):
                 masks = torch.stack(masks, dim=2).clone()
             elif isinstance(masks[0], dict) and "counts" in masks[0]:
                 # RLE interpretation
-                assert all(
-                    [(size[1], size[0]) == tuple(inst["size"]) for inst in masks]
-                )  # in RLE, height come first in "size"
+                rle_sizes = [tuple(inst["size"]) for inst in masks]
+
                 masks = mask_utils.decode(masks)  # [h, w, n]
                 masks = torch.tensor(masks).permute(2, 0, 1)  # [n, h, w]
+
+                assert rle_sizes.count(rle_sizes[0]) == len(rle_sizes), (
+                    "All the sizes must be the same size: %s" % rle_sizes
+                )
+
+                # in RLE, height come first in "size"
+                rle_height, rle_width = rle_sizes[0]
+                assert masks.shape[1] == rle_height
+                assert masks.shape[2] == rle_width
+
+                width, height = size
+                if width != rle_width or height != rle_height:
+                    masks = interpolate(
+                        input=masks[None].float(),
+                        size=(height, width),
+                        mode="bilinear",
+                        align_corners=False,
+                    )[0].type_as(masks)
             else:
                 RuntimeError(
-                    "Type of `masks[0]` could not be interpreted: %s" % type(masks)
+                    "Type of `masks[0]` could not be interpreted: %s"
+                    % type(masks)
                 )
         elif isinstance(masks, BinaryMaskList):
             # just hard copy the BinaryMaskList instance's underlying data
             masks = masks.masks.clone()
         else:
             RuntimeError(
-                "Type of `masks` argument could not be interpreted:%s" % type(masks)
+                "Type of `masks` argument could not be interpreted:%s"
+                % type(masks)
             )
 
         if len(masks.shape) == 2:
@@ -155,7 +177,9 @@ def _findContours(self):
             reshaped_contour = []
             for entity in contour:
                 assert len(entity.shape) == 3
-                assert entity.shape[1] == 1, "Hierarchical contours are not allowed"
+                assert (
+                    entity.shape[1] == 1
+                ), "Hierarchical contours are not allowed"
                 reshaped_contour.append(entity.reshape(-1).tolist())
             contours.append(reshaped_contour)
         return contours
@@ -206,7 +230,8 @@ def __init__(self, polygons, size):
 
         else:
             RuntimeError(
-                "Type of argument `polygons` is not allowed:%s" % (type(polygons))
+                "Type of argument `polygons` is not allowed:%s"
+                % (type(polygons))
             )
 
         """ This crashes the training way too many times...
@@ -278,7 +303,9 @@ def resize(self, size):
             assert isinstance(size, (int, float))
             size = size, size
 
-        ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(size, self.size))
+        ratios = tuple(
+            float(s) / float(s_orig) for s, s_orig in zip(size, self.size)
+        )
 
         if ratios[0] == ratios[1]:
             ratio = ratios[0]
@@ -349,7 +376,9 @@ def __init__(self, polygons, size):
                     type(polygons[0][0])
                 )
             else:
-                assert isinstance(polygons[0], PolygonInstance), str(type(polygons[0]))
+                assert isinstance(polygons[0], PolygonInstance), str(
+                    type(polygons[0])
+                )
 
         elif isinstance(polygons, PolygonList):
             size = polygons.size
@@ -357,7 +386,8 @@ def __init__(self, polygons, size):
 
         else:
             RuntimeError(
-                "Type of argument `polygons` is not allowed:%s" % (type(polygons))
+                "Type of argument `polygons` is not allowed:%s"
+                % (type(polygons))
             )
 
         assert isinstance(size, (list, tuple)), str(type(size))
@@ -404,7 +434,9 @@ def to(self, *args, **kwargs):
 
     def convert_to_binarymask(self):
         if len(self) > 0:
-            masks = torch.stack([p.convert_to_binarymask() for p in self.polygons])
+            masks = torch.stack(
+                [p.convert_to_binarymask() for p in self.polygons]
+            )
         else:
             size = self.size
             masks = torch.empty([0, size[1], size[0]], dtype=torch.uint8)