From 3be7b8137a700294968730d89dc560249631ec41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Hannequin?= Date: Mon, 9 Dec 2024 22:07:24 +0100 Subject: [PATCH] Write benchmark script and publish results --- Gemfile.lock | 2 + astronoby.gemspec | 1 + benchmark/README.md | 131 +++++++++++++++++++++ benchmark/benchmark.rb | 259 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 393 insertions(+) create mode 100644 benchmark/README.md create mode 100644 benchmark/benchmark.rb diff --git a/Gemfile.lock b/Gemfile.lock index c7c38e2..89ff195 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,6 +50,7 @@ GEM rubocop (>= 1.48.1, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (1.13.0) + rubyzip (2.3.2) standard (1.42.1) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) @@ -72,6 +73,7 @@ DEPENDENCIES astronoby! rake (~> 13.0) rspec (~> 3.0) + rubyzip (~> 2.3) standard (~> 1.3) BUNDLED WITH diff --git a/astronoby.gemspec b/astronoby.gemspec index 26833c5..a1f53da 100644 --- a/astronoby.gemspec +++ b/astronoby.gemspec @@ -33,6 +33,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rake", "~> 13.0" spec.add_development_dependency "rspec", "~> 3.0" + spec.add_development_dependency "rubyzip", "~> 2.3" spec.add_development_dependency "standard", "~> 1.3" # For more information and examples about making a new gem, check out our diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..cc6b9eb --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,131 @@ +# Benchmark + +This is a first attempt to benchmark the accuracy of the library. It is not +very scientific, but it gives a rough idea. + +## Method + +The goal is to answer these two questions: +- Is the library accurate enough compared to a source of truth? +- Is the library accurate enough compared with other Ruby libraries? + +The source of truth is the IMCCE, a French public +institude attached to the Paris Observatory. Their ephemerides are used by +governements and public institutions in multiple European countries, their +precesion is among the highest in the world. +They also provide web services to easily access all their data. Many thanks +for providing such high accuracy data for free. + +The other Ruby library is [sun_calc](https://github.com/fishbrain/sun_calc). + +474,336 combinations of dates, latitudes and longitudes have been used to +produce time predictions for the following events: +- sunrise +- sun's highest point +- sunset +- moonrise +- moon's highest point +- moonset + +For each combination, we first find out which of SunCalc or Astronoby is the +closest to the IMCCE. Then we calculate the difference between Astronoby and +the IMCCE, to discover if the difference is larger than the defined +threshold of 5 minutes. + +## Results + +The following output has been generated using what will be part of version 0.6. + +``` +Unarchiving sun_calc.csv.zip... +Done unarchiving sun_calc.csv.zip. +Parsing sun_calc.csv... +Done parsing sun_calc.csv. +Unarchiving imcce.csv.zip... +Done unarchiving imcce.csv.zip. +Parsing imcce.csv... +Done parsing imcce.csv. +Comparing data... +2024-01-01: Done. +2024-01-02: Done. +... +2024-12-30: Done. +2024-12-31: Done. +Done comparing data. + + +Sun rising time: +astronoby: 295395 (62.28%) +sun_calc: 99710 (21.02%) +n/a: 79231 (16.7%) + +Sun transit time: +astronoby: 434495 (91.6%) +sun_calc: 39769 (8.38%) +n/a: 72 (0.02%) + +Sun setting time: +astronoby: 358428 (75.56%) +n/a: 79231 (16.7%) +sun_calc: 36677 (7.73%) + +Moon rising time: +astronoby: 290866 (61.32%) +n/a: 113815 (23.99%) +sun_calc: 69655 (14.68%) + +Moon transit time: +astronoby: 341902 (72.08%) +n/a: 101916 (21.49%) +sun_calc: 30518 (6.43%) + +Moon setting time: +astronoby: 327099 (68.96%) +n/a: 114308 (24.1%) +sun_calc: 32929 (6.94%) + +Moon illuminated fraction: +astronoby: 474336 (100.0%) + +Sun rising time too far: +false: 452887 (95.48%) +true: 21449 (4.52%) + +Sun transit time too far: +false: 396617 (83.62%) +true: 77719 (16.38%) + +Sun setting time too far: +false: 453208 (95.55%) +true: 21128 (4.45%) + +Moon rising time too far: +false: 459044 (96.78%) +true: 15292 (3.22%) + +Moon transit time too far: +false: 384516 (81.06%) +true: 89820 (18.94%) + +Moon setting time too far: +false: 459222 (96.81%) +true: 15114 (3.19%) +``` + +## Conclusion + +As we can see, Astronoby is more accurate than SunCalc in a vast majority of +cases. When it comes to the Moon's illuminated fraction, Astronoby is always +more accurate than SunCalc. + +`n/a` values means that at least one of the three sources don't have a value +for the combination of date, latitude and longitude. This happens because +the Moon and the Sun cannot always rise, transit and set everywhere on Earth +every day of the year. Latitudes close to the poles are more likely to miss +data. + +Astronoby can be considered "good enough" for more around 90% of the cases, +which means there is still work to do if we want to always be less than 5 +minutes away from the what the IMCCE provides. We can notice that transit +times are those that experience the most significant differences. diff --git a/benchmark/benchmark.rb b/benchmark/benchmark.rb new file mode 100644 index 0000000..20db6a6 --- /dev/null +++ b/benchmark/benchmark.rb @@ -0,0 +1,259 @@ +require "astronoby" +require "csv" +require "zip" + +class Source + NAMES = [ + ASTRONOBY = "astronoby", + IMCCE = "imcce", + SUN_CALC = "sun_calc" + ].freeze + + attr_accessor :name, + :sun_rising_time, + :sun_transit_time, + :sun_setting_time, + :moon_rising_time, + :moon_transit_time, + :moon_setting_time, + :moon_illuminated_fraction +end + +class Comparison + SUN_CALC = "sun_calc" + ASTRONOBY = "astronoby" + NON_APPLICABLE = "n/a" + + TOO_FAR_THRESHOLD = 60 * 5 # 5 minutes + + attr_accessor :sources, :truth + + def initialize + @sources = [] + end + + %i[ + sun_rising_time + sun_transit_time + sun_setting_time + moon_rising_time + moon_transit_time + moon_setting_time + ].each do |attribute| + define_method(:"closest_#{attribute}") do + compare(attribute) + end + + define_method(:"#{attribute}_too_far?") do + too_far?(attribute) + end + end + + def closest_moon_illuminated_fraction + compare(:moon_illuminated_fraction) + end + + private + + def compare(attribute) + unless truth.public_send(attribute) && sources.all? { |source| source.public_send(attribute) } + return NON_APPLICABLE + end + + closest_source = sources.min_by do |source| + (truth.public_send(attribute) - source.public_send(attribute)).abs + end + + closest_source.name + end + + def too_far?(attribute) + truth_attribute = truth.public_send(attribute) + astronoby_attribute = sources + .find { _1.name == Source::ASTRONOBY } + .public_send(attribute) + + return false unless truth_attribute && astronoby_attribute + + (truth_attribute - astronoby_attribute).abs > TOO_FAR_THRESHOLD + end +end + +class Result + def initialize + @sun_rising_time = [] + @sun_transit_time = [] + @sun_setting_time = [] + @moon_rising_time = [] + @moon_transit_time = [] + @moon_setting_time = [] + @illuminated_fraction = [] + @sun_rising_time_too_far = [] + @sun_transit_time_too_far = [] + @sun_setting_time_too_far = [] + @moon_rising_time_too_far = [] + @moon_transit_time_too_far = [] + @moon_setting_time_too_far = [] + end + + def add_comparison(comparison) + @sun_rising_time << comparison.closest_sun_rising_time + @sun_transit_time << comparison.closest_sun_transit_time + @sun_setting_time << comparison.closest_sun_setting_time + @moon_rising_time << comparison.closest_moon_rising_time + @moon_transit_time << comparison.closest_moon_transit_time + @moon_setting_time << comparison.closest_moon_setting_time + @illuminated_fraction << comparison.closest_moon_illuminated_fraction + @sun_rising_time_too_far << comparison.sun_rising_time_too_far? + @sun_transit_time_too_far << comparison.sun_transit_time_too_far? + @sun_setting_time_too_far << comparison.sun_setting_time_too_far? + @moon_rising_time_too_far << comparison.moon_rising_time_too_far? + @moon_transit_time_too_far << comparison.moon_transit_time_too_far? + @moon_setting_time_too_far << comparison.moon_setting_time_too_far? + end + + def display + puts "Sun rising time:" + tally(@sun_rising_time) + puts "Sun transit time:" + tally(@sun_transit_time) + puts "Sun setting time:" + tally(@sun_setting_time) + puts "Moon rising time:" + tally(@moon_rising_time) + puts "Moon transit time:" + tally(@moon_transit_time) + puts "Moon setting time:" + tally(@moon_setting_time) + puts "Moon illuminated fraction:" + tally(@illuminated_fraction) + puts "Sun rising time too far:" + tally(@sun_rising_time_too_far) + puts "Sun transit time too far:" + tally(@sun_transit_time_too_far) + puts "Sun setting time too far:" + tally(@sun_setting_time_too_far) + puts "Moon rising time too far:" + tally(@moon_rising_time_too_far) + puts "Moon transit time too far:" + tally(@moon_transit_time_too_far) + puts "Moon setting time too far:" + tally(@moon_setting_time_too_far) + end + + private + + def tally(data) + t = data.tally + t.sort_by { |_key, value| -value }.each do |key, value| + puts "#{key}: #{value} (#{(value.to_f / t.values.sum * 100).round(2)}%)" + end + puts "\n" + end +end + +data = {} +result = Result.new + +sun_calc_zip_file = File.join(File.dirname(__FILE__), "data/sun_calc.csv.zip") +imcce_zip_file = File.join(File.dirname(__FILE__), "data/imcce.csv.zip") + +puts "Unarchiving sun_calc.csv.zip..." + +Zip::File.open(sun_calc_zip_file) do |zip_file| + puts "Done unarchiving sun_calc.csv.zip." + + csv_file = zip_file.find { |entry| entry.name.end_with?(".csv") } + break unless csv_file + + puts "Parsing sun_calc.csv..." + + csv_content = csv_file.get_input_stream.read + CSV.parse(csv_content, headers: true) do |row| + data[row["date"]] ||= {} + data[row["date"]][row["latitude"]] ||= {} + data[row["date"]][row["latitude"]][row["longitude"]] = Comparison.new.tap do |comparison| + source = Source.new.tap do |source| + source.name = Source::SUN_CALC + source.sun_rising_time = Time.new(row["sun_rising_time"]) if row["sun_rising_time"] + source.sun_transit_time = Time.new(row["sun_transit_time"]) if row["sun_transit_time"] + source.sun_setting_time = Time.new(row["sun_setting_time"]) if row["sun_setting_time"] + source.moon_rising_time = Time.new(row["moon_rising_time"]) if row["moon_rising_time"] + source.moon_transit_time = Time.new(row["moon_transit_time"]) if row["moon_transit_time"] + source.moon_setting_time = Time.new(row["moon_setting_time"]) if row["moon_setting_time"] + source.moon_illuminated_fraction = row["illuminated_fraction"].to_f + end + comparison.sources << source + end + end + + puts "Done parsing sun_calc.csv." +end + +puts "Unarchiving imcce.csv.zip..." + +Zip::File.open(imcce_zip_file) do |zip_file| + puts "Done unarchiving imcce.csv.zip." + + csv_file = zip_file.find { |entry| entry.name.end_with?(".csv") } + break unless csv_file + + puts "Parsing imcce.csv..." + + csv_content = csv_file.get_input_stream.read + CSV.parse(csv_content, headers: true) do |row| + comparison = data[row["date"]][row["latitude"]][row["longitude"]] + comparison.truth = Source.new.tap do |source| + source.name = Source::IMCCE + source.sun_rising_time = Time.new(row["sun_rising_time"] + " UTC") if row["sun_rising_time"] + source.sun_transit_time = Time.new(row["sun_transit_time"] + " UTC") if row["sun_transit_time"] + source.sun_setting_time = Time.new(row["sun_setting_time"] + " UTC") if row["sun_setting_time"] + source.moon_rising_time = Time.new(row["moon_rising_time"] + " UTC") if row["moon_rising_time"] + source.moon_transit_time = Time.new(row["moon_transit_time"] + " UTC") if row["moon_transit_time"] + source.moon_setting_time = Time.new(row["moon_setting_time"] + " UTC") if row["moon_setting_time"] + source.moon_illuminated_fraction = row["illuminated_fraction"].to_f + end + end + + puts "Done parsing imcce.csv." +end + +puts "Comparing data..." + +data.each do |date, latitudes| + latitudes.each do |latitude, longitudes| + longitudes.each do |longitude, comparison| + noon = Time.new("#{date}T12:00:00Z") + observer = Astronoby::Observer.new( + latitude: Astronoby::Angle.from_degrees(latitude.to_i), + longitude: Astronoby::Angle.from_degrees(longitude.to_i) + ) + sun = Astronoby::Sun.new(time: noon) + sun_observation_events = sun.observation_events(observer: observer) + moon = Astronoby::Moon.new(time: noon) + moon_observation_events = moon.observation_events(observer: observer) + + source = Source.new.tap do |source| + source.name = Source::ASTRONOBY + source.sun_rising_time = sun_observation_events.rising_time + source.sun_transit_time = sun_observation_events.transit_time + source.sun_setting_time = sun_observation_events.setting_time + source.moon_rising_time = moon_observation_events.rising_time + source.moon_transit_time = moon_observation_events.transit_time + source.moon_setting_time = moon_observation_events.setting_time + source.moon_illuminated_fraction = moon.illuminated_fraction + end + + comparison.sources << source + result.add_comparison(comparison) + end + end + + puts "#{date}: Done." +end + +puts "Done comparing data." +puts +puts + +puts result.display