radar

Author: Transparent6

DATASETS.md

@@ -0,0 +1,72 @@
+# How to install datasets
+
+The datasets we use are listed below:
+
+- Visual-genome [Link](https://visualgenome.org/api/v0/api_home.html)
+- HAKE [Link](http://hake-mvig.cn/home/)
+- Visual Attributes in the Wild (VAW) [Link](https://vawdataset.com/)
+- SWiG [Link](https://github.com/allenai/swig)
+
+As the images in the VAW dataset come from the Visual Genome dataset, we only need three picture folders. The file structure looks like below:
+```
+$DATASET/
+|–– hake/
+|–– swig/
+|–– vg/
+```
+
+## HAKE
+- Create a folder named `hake/` under `$DATASET/`.
+- Create `hake_images_20190730/`,`hake_images_20200614/`,`hcvrd/`,`hico_20160224_det/`,`openimages/`,`pic/` and `vcoco/` under `hake/`.
+- Download the images from the [dataset link](https://github.com/DirtyHarryLYL/HAKE/tree/master/Images#download-images-for-hake).
+- Put the images in the corresponding folder.
+
+The folder tree looks like:
+```
+hake/
+├── hake_images_20190730
+│   └── xxx.jpg
+├── hake_images_20200614
+│   └── xxx.jpg
+├── hcvrd
+│   └── xxx.jpg
+├── hico_20160224_det
+│   └── images
+│       ├── test2015
+│       │   └── xxx.jpg
+│       └── train2015
+│           └── xxx.jpg
+├── openimages
+│   └── xxx.jpg
+├── pic
+│   └── xxx.jpg
+└── vcoco
+    ├── train2014
+    |    └── xxx.jpg
+    └── val2014
+        └── xxx.jpg
+```
+## SWiG
+- Create a folder named `swig/` under `$DATASET/`.
+- Download the images from the [dataset link](https://swig-data-weights.s3.us-east-2.amazonaws.com/images_512.zip).
+
+The folder tree looks like:
+```
+swig/
+└── xxx.jpg
+```
+## VG & VAW
+- Create a folder named `vg/` under `$DATASET/`.
+- Create `VG_100K/` and `VG_100K_2/` under `vg/`.
+- Download the images from the [part1](https://cs.stanford.edu/people/rak248/VG_100K_2/images.zip) and [part2](https://cs.stanford.edu/people/rak248/VG_100K_2/images2.zip).
+- Put the images in the corresponding folder.
+
+The folder tree looks like:
+```
+vg/
+|–– VG_100K/
+    └── xxx.jpg
+|–– VG_100K_2/
+    └── xxx.jpg
+```
+

README.md

@@ -34,7 +34,7 @@ git clone https://github.com/om-ai-lab/VL-CheckList.git
 We include several representative example VLP models in the `example_models/` folder. 
 
 **1.** Define a config file 
-e.g. in [configs/sample.yam](./configs/sample.yaml)
+e.g. in [configs/sample.yaml](./configs/sample.yaml)
 ```
 MAX_NUM: 2000
 MODEL_NAME: "ViLT"

corpus.json

@@ -0,0 +1,65 @@
+{
+    "large":{
+        "vg":["Object_Size_large_vg_obj","Object_Size_large_vg_subj"],
+        "swig":["Object_Size_large_swig_agent","Object_Size_large_swig_destination","Object_Size_large_swig_item","Object_Size_large_swig_tool"],
+        "hake":["Object_Size_large_hake"]
+    },
+
+    "medium":{
+        "vg":["Object_Size_medium_vg_obj","Object_Size_medium_vg_subj"],
+        "swig":["Object_Size_medium_swig_agent","Object_Size_medium_swig_destination","Object_Size_medium_swig_item","Object_Size_medium_swig_tool"],
+        "hake":["Object_Size_medium_hake"]    
+    },
+
+    "small":{
+        "vg":["Object_Size_small_vg_obj","Object_Size_small_vg_subj"],
+        "swig":["Object_Size_small_swig_agent","Object_Size_small_swig_destination","Object_Size_small_swig_item","Object_Size_small_swig_tool"],
+        "hake":["Object_Size_small_hake"]    
+    },
+
+    "center":{
+        "vg":["Object_Location_center_vg_obj","Object_Location_center_vg_subj"],
+        "swig":["Object_Location_center_swig_agent","Object_Location_center_swig_destination","Object_Location_center_swig_item","Object_Location_center_swig_tool"],
+        "hake":["Object_Location_center_hake"]
+    },
+    "mid":{
+        "vg":["Object_Location_mid_vg_obj","Object_Location_mid_vg_subj"],
+        "swig":["Object_Location_mid_swig_agent","Object_Location_mid_swig_destination","Object_Location_mid_swig_item","Object_Location_mid_swig_tool"],
+        "hake":["Object_Location_mid_hake"]
+    },
+    "margin":{
+        "vg":["Object_Location_margin_vg_obj","Object_Location_margin_vg_subj"],
+        "swig":["Object_Location_margin_swig_agent","Object_Location_margin_swig_destination","Object_Location_margin_swig_item","Object_Location_margin_swig_tool"],
+        "hake":["Object_Location_margin_hake"]
+    },
+
+    "color":{
+        "vg":["Attribute_color_vg"],
+        "vaw":["Attribute_color_vaw"]
+    },
+    "material":{
+        "vg":["Attribute_material_vg"],
+        "vaw":["Attribute_material_vaw"]
+    },
+    "action_a":{
+        "vg":["Attribute_action_vg"],
+        "vaw":["Attribute_action_vaw"]
+    },
+    "size":{
+        "vg":["Attribute_size_vg"],
+        "vaw":["Attribute_size_vaw"]
+    },
+    "state":{
+        "vg":["Attribute_state_vg"],
+        "vaw":["Attribute_state_vaw"]
+    },
+
+    "action_r":{
+        "vg":["Relation_action_vg"],
+        "swig":["Relation_action_swig"],
+        "hake":["Relation_action_hake"]
+    },
+    "spatial":{
+        "vg":["Relation_spatial_vg"]
+    }
+}

radar.py

@@ -0,0 +1,168 @@
+import numpy as np
+
+import matplotlib.pyplot as plt
+from matplotlib.patches import Circle, RegularPolygon
+from matplotlib.path import Path
+from matplotlib.projections.polar import PolarAxes
+from matplotlib.projections import register_projection
+from matplotlib.spines import Spine
+from matplotlib.transforms import Affine2D
+import matplotlib
+import random
+import os
+import json
+
+def radar_factory(num_vars, frame='circle'):
+    """Create a radar chart with `num_vars` axes.
+
+    This function creates a RadarAxes projection and registers it.
+
+    Parameters
+    ----------
+    num_vars : int
+        Number of variables for radar chart.
+    frame : {'circle' | 'polygon'}
+        Shape of frame surrounding axes.
+
+    """
+    # calculate evenly-spaced axis angles
+    theta = np.linspace(0, 2 * np.pi, num_vars, endpoint=False)
+
+    class RadarAxes(PolarAxes):
+
+        name = 'radar'
+
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            # rotate plot such that the first axis is at the top
+            self.set_theta_zero_location('N')
+
+        def fill(self, *args, closed=True, **kwargs):
+            """Override fill so that line is closed by default"""
+            return super().fill(closed=closed, *args, **kwargs)
+
+        def plot(self, *args, **kwargs):
+            """Override plot so that line is closed by default"""
+            lines = super().plot(*args, **kwargs)
+            for line in lines:
+                self._close_line(line)
+
+        def _close_line(self, line):
+            x, y = line.get_data()
+            # FIXME: markers at x[0], y[0] get doubled-up
+            if x[0] != x[-1]:
+                x = np.concatenate((x, [x[0]]))
+                y = np.concatenate((y, [y[0]]))
+                line.set_data(x, y)
+
+        def set_varlabels(self, labels):
+            self.set_thetagrids(np.degrees(theta), labels)
+
+        def _gen_axes_patch(self):
+            # The Axes patch must be centered at (0.5, 0.5) and of radius 0.5
+            # in axes coordinates.
+            if frame == 'circle':
+                return Circle((0.5, 0.5), 0.5)
+            elif frame == 'polygon':
+                return RegularPolygon((0.5, 0.5), num_vars,
+                                      radius=.5, edgecolor="k")
+            else:
+                raise ValueError("unknown value for 'frame': %s" % frame)
+
+        def draw(self, renderer):
+            """ Draw. If frame is polygon, make gridlines polygon-shaped """
+            if frame == 'polygon':
+                gridlines = self.yaxis.get_gridlines()
+                for gl in gridlines:
+                    gl.get_path()._interpolation_steps = num_vars
+            super().draw(renderer)
+
+        def _gen_axes_spines(self):
+            if frame == 'circle':
+                return super()._gen_axes_spines()
+            elif frame == 'polygon':
+                # spine_type must be 'left'/'right'/'top'/'bottom'/'circle'.
+                spine = Spine(axes=self,
+                              spine_type='circle',
+                              path=Path.unit_regular_polygon(num_vars))
+                # unit_regular_polygon gives a polygon of radius 1 centered at
+                # (0, 0) but we want a polygon of radius 0.5 centered at (0.5,
+                # 0.5) in axes coordinates.
+                spine.set_transform(Affine2D().scale(.5).translate(.5, .5)
+                                    + self.transAxes)
+
+                return {'polar': spine}
+            else:
+                raise ValueError("unknown value for 'frame': %s" % frame)
+
+    register_projection(RadarAxes)
+    return theta
+
+def generate_chart(output_folder,corpus_path,task,chart_type,models):
+    """
+    output_folder : str
+        The path of folder that saves all evaluation results.
+    corpus_path : str
+        The path of json file that records corresponding file names of all metrics
+    task : {'itm' | 'itc'}
+        Task type of output result.
+    chart_type : str
+        Type of chart.
+    models : list
+        List of model name.
+
+    """    
+    m = json.load(open(corpus_path))
+    arrs = []
+    colors = ["cyan", "magenta", "crimson", "orange","blue"]
+    for model in models:
+        filepath = os.path.join(output_folder,model,task)
+        score_list = []
+
+        for item in m.keys():
+            data_num = len(m[item].keys())
+            data_score = []
+            for data in m[item].keys():
+                score = 0
+                file_num = len(m[item][data])
+                for file in m[item][data]:
+                    json_name = os.path.join(filepath,f"{file}.json")
+                    if not os.path.exists(json_name):
+                        print(f"{file} has not been evaluated.")
+                        return
+                    else:
+                        m1 = json.load(open(json_name))
+                        score += m1["total_acc"]
+                data_score.append(score/file_num)
+            score_list.append(sum(data_score)/data_num)
+        arrs.append(score_list)
+
+    fig_ = plt.figure(figsize=(25, 8))
+    models_name = [models]
+    for i in range(len(models_name)):
+        data = [['O-Large', 'O-Medium', 'O-Small', 'O-Center', 'O-Mid', 'O-Margin', 'A-Color', 'A-Material', 'A-Size',
+                "A-State", "A-Action", "R-action", "R-spatial"],
+                ('', arrs)]
+        N = len(data[0])
+        theta = radar_factory(N, frame='polygon')
+
+        spoke_labels = data.pop(0)
+        title, case_data = data[0]
+        ax = fig_.add_subplot(f'13{i+1}', projection='radar')
+
+        ax.set_rgrids([30, 40, 50, 55, 60, 65, 70, 75, 80, 85, 90])
+        lines = []
+        labels = models_name[i]
+        for i, d in enumerate(case_data):
+            c = colors.pop()
+            line = ax.plot(theta, d, label=labels[i], color=c)
+            lines.append(line)
+            ax.fill(theta, d, alpha=0.25, color=c)
+        ax.set_varlabels(spoke_labels)
+        plt.legend(loc="best", bbox_to_anchor=(1, 1))
+
+    plt.show()
+
+if __name__ == "__main__":
+
+    generate_chart('./output','corpus.json','itc',chart_type='radar',models=['vilt','albef'])