-
Notifications
You must be signed in to change notification settings - Fork 3
/
plotting_commands.txt
36 lines (27 loc) · 4.92 KB
/
plotting_commands.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# Effect of symmetries
incal-experiments smt-lib-benchmark analyze --dirs remote_res/smt_lib_benchmark/qf_lra_results_plain/ remote_res/smt_lib_benchmark/qf_lra_results_m/ remote_res/smt_lib_benchmark/qf_lra_results_n/ remote_res/smt_lib_benchmark/qf_lra_results_v/ remote_res/smt_lib_benchmark/qf_lra_results_mvn/ remote_res/smt_lib_benchmark/qf_lra_results_h --res_path remote_res show -p -t duration -g k learner -o ly "Duration (s)" lx "Number of learned clauses (k)" steps_x 3 error 0 -w ../incal_mlj/figures/symmetries_benchmark.png
# Counts
incal-experiments smt-lib-benchmark analyze --dirs remote_res/smt_lib_benchmark/qf_lra_results_plain/ remote_res/smt_lib_benchmark/qf_lra_results_m/ remote_res/smt_lib_benchmark/qf_lra_results_n/ remote_res/smt_lib_benchmark/qf_lra_results_v/ remote_res/smt_lib_benchmark/qf_lra_results_mvn/ remote_res/smt_lib_benchmark/qf_lra_results_h --res_path remote_res show -p -a count -t id -g k learner -o ly "Number of runs" lx "Number of learned clauses (k)" steps_x 3 error 0
# Learned halfspaces
incal-experiments smt-lib-benchmark analyze --dirs remote_res/smt_lib_benchmark/qf_lra_results_plain/ remote_res/smt_lib_benchmark/qf_lra_results_m/ remote_res/smt_lib_benchmark/qf_lra_results_n/ remote_res/smt_lib_benchmark/qf_lra_results_v/ remote_res/smt_lib_benchmark/qf_lra_results_mvn/ remote_res/smt_lib_benchmark/qf_lra_results_h --res_path remote_res show -p -t h -g k learner -o error 1
# Effect of dt
incal-experiments smt-lib-benchmark analyze --dirs remote_res/smt_lib_benchmark/qf_lra_results_plain/ remote_res/smt_lib_benchmark/qf_lra_results_dt/ remote_res/smt_lib_benchmark/qf_lra_results_mvn remote_res/smt_lib_benchmark/qf_lra_results_dt_mvn --res_path remote_res show -p -t duration -g k selection_strategy learner -o ly "Duration (s)" lx "Number of learned clauses (k)" steps_x 3 error 0 -w ../incal_mlj/figures/dt_benchmark.png
# --- Exploring H
# Duration over increasing original h
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t duration -g original_h selection_strategy learner -o ly "Duration (s)" lx "Number of inequalities in ground truth (h)" -w ../incal_mlj/figures/time_over_oh_synthetic.png
# Learned h over original h
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t h -g original_h selection_strategy learner -o ly "Number of learned hyperplanes (h)" lx "Number of inequalities in ground truth (h)" -w ../incal_mlj/figures/h_over_oh_synthetic.png
# Duration over learned h
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t duration -g h selection_strategy learner -o ly "Duration (s)" lx "Number of learned inequalities (h)" -w ../incal_mlj/figures/time_over_h_synthetic.png
# Accuracy over learned h
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain res/synthetic/output/hh_mvn res/synthetic/output/hh_dt res/synthetic/output/hh_dt_mvn res/synthetic/output/hh_dt_mn --res_path res show -p -t accuracy_approx -g h selection_strategy learner -o ly "Accuracy on test set" lx "Number of learned inequalities (h)" y_lim "0.5:1" legend_pos "lower center" -w ../incal_mlj/figures/acc_over_h_synthetic.png
# Duration over positive ratio
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/hh_plain --res_path res show -p -t duration -g pos_rate__batch0.1 h -o ly "Duration (s)" lx "Positive rate" -e "k!=2" "h<5" -w ../incal_mlj/figures/time_over_pos_rate.png
# --- Exploring L
# Timeouts over increasing original l
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/ll_plain --res_path res show -p -t executed -g original_l learner -o error 0 y_lim 0:1 lx "Number of literals in ground truth" ly "Fraction of experiments completed within time limit" steps_x 5 -w ../incal_mlj/figures/ex_over_l_synthetic.png
# --- Exploring K
# Timeouts over increasing original l
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/kk_plain res/synthetic/output/kk_dt_mvn --res_path res show -p -t executed -g original_k learner -o error 0 y_lim 0:1 lx "Number of clauses in ground truth" ly "Fraction of experiments completed within time limit" steps_x 5 -w ../incal_mlj/figures/ex_over_k_synthetic.png
# Learned k over original k
incal-experiments smt-lib-benchmark analyze --dirs res/synthetic/output/kk_plain res/synthetic/output/kk_dt_mvn --res_path res show -p -t k -g original_k selection_strategy learner -o ly "Number of learned clauses (k)" lx "Number of clauses in ground truth (k)" -e "executed=0" -w ../incal_mlj/figures/k_over_original_k_synthetic.png