- Enhancements for analysis and plotting

Author: samuelkolb

incal/experiments/analyze.py

@@ -4,6 +4,7 @@
 
 from typing import List
 
+import numpy as np
 import pickledb
 from pywmi import RejectionEngine, nested_to_smt, import_domain
 from pywmi.domain import Density, Domain
@@ -61,16 +62,33 @@ def get_bound(experiment):
     def get_db_synthetic(experiment):
         return get_synthetic_db(os.path.dirname(experiment.parameters.original_values["domain"]))
 
+    @staticmethod
+    def original_k(experiment):
+        db = Properties.get_db_synthetic(experiment)
+        name = Properties.to_synthetic_name(experiment.imported_from_file)
+        return db.get(name)["generation"]["k"]
+
     @staticmethod
     def original_h(experiment):
         db = Properties.get_db_synthetic(experiment)
         name = Properties.to_synthetic_name(experiment.imported_from_file)
         return db.get(name)["generation"]["h"]
 
+    @staticmethod
+    def original_l(experiment):
+        db = Properties.get_db_synthetic(experiment)
+        name = Properties.to_synthetic_name(experiment.imported_from_file)
+        return db.get(name)["generation"]["l"]
+
     @staticmethod
     def executed(experiment):
         return 1 if experiment.results.duration is not None else 0
 
+    @staticmethod
+    def positive_ratio(experiment):
+        labels = np.load(experiment.parameters.original_values["labels"])
+        return sum(labels) / len(labels)
+
     @staticmethod
     def accuracy_approx(experiment):
         key = "accuracy_approx:{}".format(experiment.imported_from_file)
@@ -101,7 +119,10 @@ def accuracy_approx(experiment):
 def register_derived(experiment):
     experiment.register_derived("accuracy_approx", Properties.accuracy_approx)
     experiment.register_derived("original_h", Properties.original_h)
+    experiment.register_derived("original_l", Properties.original_l)
+    experiment.register_derived("original_k", Properties.original_k)
     experiment.register_derived("executed", Properties.executed)
+    experiment.register_derived("pos_rate", Properties.positive_ratio)
     return experiment
 
 

incal/util/plot.py

@@ -51,7 +51,7 @@ def gen_markers(self):
         return [next(iterator) for _ in range(len(self.data))]
 
     def render(self, ax, lines=True, log_x=True, log_y=True, label_x=None, label_y=None, legend_pos=None,
-               x_ticks=None):
+               x_ticks=None, y_ticks=None):
 
         plots = []
         colors = self.gen_colors()
@@ -65,6 +65,7 @@ def render(self, ax, lines=True, log_x=True, log_y=True, label_x=None, label_y=N
         plot_format = "scatter"
         show_error = True
         steps_x = None
+        steps_y = None
 
         cache = None
         for plot_option in self.plot_options or ():
@@ -86,8 +87,18 @@ def render(self, ax, lines=True, log_x=True, log_y=True, label_x=None, label_y=N
                     label_y = plot_option
                 elif cache == "steps_x":
                     steps_x = int(plot_option)
+                elif cache == "steps_y":
+                    steps_y = int(plot_option)
                 elif cache == "plot_extra":
                     plot_extra = plot_option
+                elif cache == "x_lim":
+                    parts = plot_option.split(":")
+                    limits = (float(parts[0]), float(parts[1]))
+                    self.x_lim(limits)
+                elif cache == "y_lim":
+                    parts = plot_option.split(":")
+                    limits = (float(parts[0]), float(parts[1]))
+                    self.y_lim(limits)
                 cache = None
 
         min_x, max_x, min_y, max_y = numpy.infty, -numpy.infty, numpy.infty, -numpy.infty
@@ -143,13 +154,15 @@ def render(self, ax, lines=True, log_x=True, log_y=True, label_x=None, label_y=N
         if label_x is not None:
             ax.set_xlabel(label_x)
 
-        # ax.set_ylim((0, 2))
-
         if steps_x is not None:
             x_ticks = numpy.linspace(min_x, max_x, steps_x)
+        if steps_y is not None:
+            y_ticks = numpy.linspace(min_y, max_y, steps_y)
         # x_ticks = [1, 2, 3]
         if x_ticks is not None:
             ax.xaxis.set_ticks(x_ticks)
+        if y_ticks is not None:
+            ax.yaxis.set_ticks(y_ticks)
 
     def plot(self, filename=None, size=None, **kwargs):
         fig = plt.figure()

plotting_commands.txt

@@ -12,10 +12,25 @@ incal-experiments smt-lib-benchmark analyze --dirs remote_res/smt_lib_benchmark/
 
 # --- Exploring H
 # Duration over increasing original h
-incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn --res_path res show -p -t duration -g original_h selection_strategy learner -o ly "Duration (s)" lx "Number of inequalities in ground truth (h)"
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t duration -g original_h selection_strategy learner -o ly "Duration (s)" lx "Number of inequalities in ground truth (h)" -w ../incal_mlj/figures/time_over_oh_synthetic.png
 # Learned h over original h
-incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn --res_path res show -p -t h -g original_h selection_strategy learner -o ly "Number of learned hyperplanes (h)" lx "Number of inequalities in ground truth (h)"
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t h -g original_h selection_strategy learner -o ly "Number of learned hyperplanes (h)" lx "Number of inequalities in ground truth (h)" -w ../incal_mlj/figures/h_over_oh_synthetic.png
 # Duration over learned h
-incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn --res_path res show -p -t duration -g h selection_strategy learner -o ly "Duration (s)" lx "Number of learned inequalities (h)"
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t duration -g h selection_strategy learner -o ly "Duration (s)" lx "Number of learned inequalities (h)" -w ../incal_mlj/figures/time_over_h_synthetic.png
 # Accuracy over learned h
-incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn --res_path res show -p -t accuracy_approx -g h selection_strategy learner -o ly "Accuracy on test set" lx "Number of learned inequalities (h)"
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t accuracy_approx -g h selection_strategy learner -o ly "Accuracy on test set" lx "Number of learned inequalities (h)" y_lim "0.5:1" legend_pos "lower center" -w ../incal_mlj/figures/acc_over_h_synthetic.png
+
+# Duration over positive ratio
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain --res_path res show -p -t duration -g pos_rate__batch0.1 h -o ly "Duration (s)" lx "Positive rate" -e "k!=2" "h<5" -w ../incal_mlj/figures/time_over_pos_rate.png
+
+
+# --- Exploring L
+# Timeouts over increasing original l
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/ll_plain --res_path res show -p -t executed -g original_l learner -o error 0 y_lim 0:1 lx "Number of literals in ground truth" ly "Fraction of experiments completed within time limit" steps_x 5 -w ../incal_mlj/figures/ex_over_l_synthetic.png
+
+
+# --- Exploring K
+# Timeouts over increasing original l
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/kk_plain res/synthetic/output/kk_dt_mvn --res_path res show -p -t executed -g original_k learner -o error 0 y_lim 0:1 lx "Number of clauses in ground truth" ly "Fraction of experiments completed within time limit" steps_x 5 -w ../incal_mlj/figures/ex_over_k_synthetic.png
+# Learned k over original k
+incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/kk_plain res/synthetic/output/kk_dt_mvn --res_path res show -p -t k -g original_k selection_strategy learner -o ly "Number of learned clauses (k)" lx "Number of clauses in ground truth (k)" -e "executed=0" -w ../incal_mlj/figures/k_over_original_k_synthetic.png