From 088f8c96272999e3889a713c25f01ff4d2fa70e9 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Fri, 14 Jun 2024 17:08:11 -0400 Subject: [PATCH 1/5] use GraphQL to check snapshot --- openneuro/app/models/open_neuro.rb | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 1797495..d3b3247 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -40,6 +40,8 @@ class OpenNeuro DATALAD_REPO_URL_PREFIX = 'https://github.com/OpenNeuroDatasets' GITHUB_VALIDATION_URL = 'https://api.github.com/repos/OpenNeuroDatasets/:name/git/ref/tags/:version' + OPENNEURO_API_URL = 'https://openneuro.org/crn/graphql' + # Creates an OpenNeuro object that represents # the dataset internally as a pair, a WorkGroup @@ -332,6 +334,48 @@ def self.data_provider_name_builder(name, version) def self.valid_name_and_version?(name, version) return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ + query = '{snapshot(datasetId:"%s", tag:"%s"){id}}' % [name, version] + + response = Typhoeus.post(OPENNEURO_API_URL, + {:body => JSON.pretty_generate({"query" => query}) + }, + :headers => { :Accept => 'application/json' } + ) + + + # Parse the response + body = response.response_body + json = JSON.parse(body) + return ! json["errors"] + rescue => ex + Rails.logger.error "OpenNeuro API request failed: #{ex.class} #{ex.message}" + return nil + end + + + # Validation of a pair [ dataset, tag ] on github performed with: + # + # curl -H "Accept: application/vnd.github+json" + # -H "X-GitHub-Api-Version: 2022-11-28" + # "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/ref/tags/2.4.0" + # + # The typical response is like this: + # + # { + # "ref": "refs/tags/2.4.0", + # "node_id": "REF_kwDOK8fpRq9yZWZzL3RhZ3MvMi40LjA", + # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/refs/tags/2.4.0", + # "object": { + # "sha": "1aa6d3a098d16009d39adde6a7abe1c34d4b07d6", + # "type": "commit", + # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/commits/1aa6d3a098d16009d39adde6a7abe1c34d4b07d6" + # } + # } + # + # The method just returns true or false. + def self.valid_name_and_github_tag?(name, version) + return false unless name =~ /\Ads\d+\z/ + return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ validation_url = GITHUB_VALIDATION_URL .sub(':name', name ) From a97172f393aadbe0907a6530202ce422bc8a0f37 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Fri, 14 Jun 2024 18:04:44 -0400 Subject: [PATCH 2/5] use GraphQL to check OpenNeuro snapshot - add headers --- openneuro/app/models/open_neuro.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index d3b3247..04a1b44 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -332,27 +332,26 @@ def self.data_provider_name_builder(name, version) # # The method just returns true or false. def self.valid_name_and_version?(name, version) - return false unless name =~ /\Ads\d+\z/ + return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ query = '{snapshot(datasetId:"%s", tag:"%s"){id}}' % [name, version] response = Typhoeus.post(OPENNEURO_API_URL, - {:body => JSON.pretty_generate({"query" => query}) - }, - :headers => { :Accept => 'application/json' } + :body => JSON.pretty_generate({ "query" => query }), + :headers => { 'Content-Type' => 'application/json', + 'Accept' => 'application/json' + } ) - # Parse the response - body = response.response_body - json = JSON.parse(body) - return ! json["errors"] + body = response.response_body + json = JSON.parse(body) + return !json["errors"] rescue => ex Rails.logger.error "OpenNeuro API request failed: #{ex.class} #{ex.message}" return nil end - # Validation of a pair [ dataset, tag ] on github performed with: # # curl -H "Accept: application/vnd.github+json" From b5f94e2a56c2fd5e9c1a5fd567b1d545ac7efcfb Mon Sep 17 00:00:00 2001 From: Serge Date: Sun, 16 Jun 2024 08:59:23 -0400 Subject: [PATCH 3/5] Update open_neuro.rb copyright year --- openneuro/app/models/open_neuro.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 04a1b44..47cb475 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -2,7 +2,7 @@ # # CBRAIN Project # -# Copyright (C) 2008-2023 +# Copyright (C) 2008-2024 # The Royal Institution for the Advancement of Learning # McGill University # @@ -42,7 +42,6 @@ class OpenNeuro GITHUB_VALIDATION_URL = 'https://api.github.com/repos/OpenNeuroDatasets/:name/git/ref/tags/:version' OPENNEURO_API_URL = 'https://openneuro.org/crn/graphql' - # Creates an OpenNeuro object that represents # the dataset internally as a pair, a WorkGroup # and a DataladDataProvider. The naming of these From 8f3afa090d7f9d670dc375c184b38d0aad641c22 Mon Sep 17 00:00:00 2001 From: Serge Date: Mon, 17 Jun 2024 16:46:08 -0400 Subject: [PATCH 4/5] change comments for name and version validation method --- openneuro/app/models/open_neuro.rb | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 47cb475..342866a 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -310,26 +310,15 @@ def self.data_provider_name_builder(name, version) "OpenNeuro-#{name}-#{version.gsub('.','_')}" end - # Validation of a pair [ dataset, version ] performed with: + # Validation of a pair [ dataset, version ] with OpenNeuro GraphQL API + # + # The result is sometimes different + # "https://api.github.com/repos/OpenNeuroDatasets/ # - # curl -H "Accept: application/vnd.github+json" - # -H "X-GitHub-Api-Version: 2022-11-28" - # "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/ref/tags/2.4.0" - # - # The typical response is like this: - # - # { - # "ref": "refs/tags/2.4.0", - # "node_id": "REF_kwDOK8fpRq9yZWZzL3RhZ3MvMi40LjA", - # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/refs/tags/2.4.0", - # "object": { - # "sha": "1aa6d3a098d16009d39adde6a7abe1c34d4b07d6", - # "type": "commit", - # "url": "https://api.github.com/repos/OpenNeuroDatasets/ds004906/git/commits/1aa6d3a098d16009d39adde6a7abe1c34d4b07d6" - # } - # } - # - # The method just returns true or false. + # Some datasets or dataset versions, availabe on OpenNeuro portal or with OpenNeuro client are not always + # on GitHub + # + # At the moment the method just returns true or false. def self.valid_name_and_version?(name, version) return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ From 1b501e3a755a7a9748d6d7656b672e1dd6e86a51 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Tue, 16 Jul 2024 07:07:57 -0400 Subject: [PATCH 5/5] different message depending for OpenNeuro sets which are not yet on Github --- openneuro/app/controllers/open_neuro_controller.rb | 8 ++++++-- openneuro/app/models/open_neuro.rb | 12 +++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/openneuro/app/controllers/open_neuro_controller.rb b/openneuro/app/controllers/open_neuro_controller.rb index 24144d4..4b902be 100644 --- a/openneuro/app/controllers/open_neuro_controller.rb +++ b/openneuro/app/controllers/open_neuro_controller.rb @@ -31,8 +31,12 @@ def show name = params[:name] version = params[:version] @open_neuro = OpenNeuro.find(name,version) - if ! @open_neuro.valid_name_and_version? - message = "The OpenNeuro dataset name '#{name}' with version '#{version}' is not valid." + if ! @open_neuro.on_github_mirror? + if @open_neuro.valid_name_and_version? + message = "The OpenNeuro dataset name '#{name}' with version '#{version}' is not yet on Github. Please try latter or contact us." + else + message = "The OpenNeuro dataset name '#{name}' with version '#{version}' does not exists." + end flash.now[:error] = message if ! current_user flash[:error] = message if current_user redirect_to :action => :select if current_user diff --git a/openneuro/app/models/open_neuro.rb b/openneuro/app/models/open_neuro.rb index 342866a..ee6c59e 100644 --- a/openneuro/app/models/open_neuro.rb +++ b/openneuro/app/models/open_neuro.rb @@ -149,6 +149,11 @@ def valid_name_and_version? self.class.valid_name_and_version?(self.name,self.version) end + # Instance method of the class method + def on_github_mirror? + self.class.on_github_mirror?(self.name,self.version) + end + ############################################################ # Meta data persistence helpers; here we store and retrieve # miscellanous values used for tracking the progres of @@ -312,15 +317,16 @@ def self.data_provider_name_builder(name, version) # Validation of a pair [ dataset, version ] with OpenNeuro GraphQL API # - # The result is sometimes different + # The result is sometimes different from # "https://api.github.com/repos/OpenNeuroDatasets/ # # Some datasets or dataset versions, availabe on OpenNeuro portal or with OpenNeuro client are not always # on GitHub # # At the moment the method just returns true or false. + # Failures/rate restriction of Github API are not handled def self.valid_name_and_version?(name, version) - return false unless name =~ /\Ads\d+\z/ + return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/ query = '{snapshot(datasetId:"%s", tag:"%s"){id}}' % [name, version] @@ -360,7 +366,7 @@ def self.valid_name_and_version?(name, version) # } # # The method just returns true or false. - def self.valid_name_and_github_tag?(name, version) + def self.on_github_mirror?(name, version) return false unless name =~ /\Ads\d+\z/ return false unless version =~ /\A[a-z0-9][\w\.\-]+\z/