From f36498690c3bb81f547d78d2206ca858d233994c Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 15 Apr 2024 08:54:09 -0700 Subject: [PATCH 01/71] [full ci] CV2-4126 Run Video Integration Tests From a804fe6de81801e6fe679d1b17e49ecab5debd9d Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 26 Jun 2024 11:30:18 -0700 Subject: [PATCH 02/71] [full ci] add redis as dependency on presto --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f1b18b2a0e..128d2f605d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,6 +159,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-image: build: presto platform: linux/amd64 @@ -174,6 +175,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-audio: build: presto platform: linux/amd64 @@ -189,6 +191,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-video: build: presto platform: linux/amd64 @@ -204,6 +207,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis alegre: build: alegre ports: From b0e48efacccb5ae56398a657e980060ac574e84c Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 16 Sep 2024 08:26:15 -0700 Subject: [PATCH 03/71] [full ci] integration tests on presto integration for text From 22efc1978549677b585ab0e9033cf4ab1b1ca06a Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 16 Sep 2024 10:34:56 -0700 Subject: [PATCH 04/71] [full ci] Update docker-compose.yml --- docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 128d2f605d..f1b18b2a0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-image: build: presto platform: linux/amd64 @@ -175,7 +174,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-audio: build: presto platform: linux/amd64 @@ -191,7 +189,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-video: build: presto platform: linux/amd64 @@ -207,7 +204,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis alegre: build: alegre ports: From 20a4f12b2054847d4ae5ba177f27d98830499ee7 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 16 Sep 2024 15:13:54 -0700 Subject: [PATCH 05/71] [full ci] test again after updates From 8d39278ebf6106af9842635f547a5f4f62a2efe3 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 16 Sep 2024 15:21:08 -0700 Subject: [PATCH 06/71] [full ci] test again after updates From 896eac88d0311aebd10b4d2efddc9a65d2f03974 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 15 Apr 2024 08:54:09 -0700 Subject: [PATCH 07/71] [full ci] CV2-4126 Run Video Integration Tests From 2160c4a7539e4752cdfa4bf06e0496b052ecaa1f Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 26 Jun 2024 11:30:18 -0700 Subject: [PATCH 08/71] [full ci] add redis as dependency on presto --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f1b18b2a0e..128d2f605d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,6 +159,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-image: build: presto platform: linux/amd64 @@ -174,6 +175,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-audio: build: presto platform: linux/amd64 @@ -189,6 +191,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-video: build: presto platform: linux/amd64 @@ -204,6 +207,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis alegre: build: alegre ports: From d777337b2e54e7edf434189cbf435adf5e8b20fa Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 18 Sep 2024 07:05:55 -0700 Subject: [PATCH 09/71] [full ci] test again after updates From dde58eec7c6bbfc7e1c77176c793696e5344a109 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 18 Sep 2024 09:10:19 -0700 Subject: [PATCH 10/71] Update docker-compose.yml --- docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 128d2f605d..f1b18b2a0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-image: build: presto platform: linux/amd64 @@ -175,7 +174,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-audio: build: presto platform: linux/amd64 @@ -191,7 +189,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-video: build: presto platform: linux/amd64 @@ -207,7 +204,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis alegre: build: alegre ports: From fac1e00cb7b4dac75363bb6c2a1169c7e1a02d21 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 18 Sep 2024 09:19:27 -0700 Subject: [PATCH 11/71] [full ci] test again after updates From 5c66e0cb2fa1956a257e15655065f8225c379895 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 18 Sep 2024 10:18:35 -0700 Subject: [PATCH 12/71] [full ci] test again after updates --- docker-compose.yml | 15 +++++++++++++++ docker-test.yml | 3 +++ 2 files changed, 18 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f1b18b2a0e..ab870ac1cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -204,6 +204,21 @@ services: depends_on: elasticmq: condition: service_healthy + presto-mean-tokens: + build: presto + platform: linux/amd64 + volumes: + - "./presto:/app" + env_file: + - presto/.env_file + environment: + ROLE: worker + MODEL_NAME: mean_tokens.Model + networks: + - dev + depends_on: + elasticmq: + condition: service_healthy alegre: build: alegre ports: diff --git a/docker-test.yml b/docker-test.yml index 4d0e04be16..9df241aa11 100644 --- a/docker-test.yml +++ b/docker-test.yml @@ -54,6 +54,9 @@ services: presto-video: environment: BOILERPLATE_ENV: test + presto-mean-tokens: + environment: + BOILERPLATE_ENV: test web: environment: PLATFORM: web From cdcedc63886c6b35bec3853e731aab05cd914a31 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Thu, 19 Sep 2024 07:16:46 -0700 Subject: [PATCH 13/71] [full ci] relax mean-tokens constraint for now --- docker-compose.yml | 15 --------------- docker-test.yml | 3 --- 2 files changed, 18 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ab870ac1cd..f1b18b2a0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -204,21 +204,6 @@ services: depends_on: elasticmq: condition: service_healthy - presto-mean-tokens: - build: presto - platform: linux/amd64 - volumes: - - "./presto:/app" - env_file: - - presto/.env_file - environment: - ROLE: worker - MODEL_NAME: mean_tokens.Model - networks: - - dev - depends_on: - elasticmq: - condition: service_healthy alegre: build: alegre ports: diff --git a/docker-test.yml b/docker-test.yml index 9df241aa11..4d0e04be16 100644 --- a/docker-test.yml +++ b/docker-test.yml @@ -54,9 +54,6 @@ services: presto-video: environment: BOILERPLATE_ENV: test - presto-mean-tokens: - environment: - BOILERPLATE_ENV: test web: environment: PLATFORM: web From f1bd1b0fdc9b018048fc6164531dd4a2b20379c4 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Thu, 19 Sep 2024 09:32:00 -0700 Subject: [PATCH 14/71] [full ci] test again after updates From 418501275cd17c30a057d9e8306def3613c0c1be Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Fri, 20 Sep 2024 07:25:27 -0700 Subject: [PATCH 15/71] [full ci] comment out other tests and watch alegre --- build.sh | 2 +- test/spec/similarity_spec.rb | 226 +++++++++++++++++------------------ 2 files changed, 114 insertions(+), 114 deletions(-) diff --git a/build.sh b/build.sh index 8ae56ac434..035d92adc4 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ else # Uncomment to debug Check API and Alegre. Warning: This can lead to Travis error "The job exceeded the maximum log length, and has been terminated.". # tail -f check-api/log/test.log & # docker-compose logs -f api & - # docker-compose logs -f alegre & + docker-compose logs -f alegre & # docker-compose logs -f presto-server & # docker-compose logs -f presto-image & # docker-compose logs -f presto-audio & diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index b043d45461..5811ab6573 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -1,68 +1,68 @@ shared_examples 'similarity' do - it 'should import and export items', bin4: true do - api_create_team_claims_sources_and_redirect_to_all_items({ count: 3 }) - verbose_wait 2 # Wait for the items to be indexed in ElasticSearch - wait_for_selector('.search__results-heading') - wait_for_selector('.cluster-card').click - wait_for_selector('#media-similarity__add-button').click - - # Merge similar items - add_related_item('Claim 0') - wait_for_selector('.media__relationship') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - expect(@driver.page_source.include?('Media')).to be(true) - end - - it 'should pin and remove similarity items', bin4: true do - data = api_create_team_and_bot - pm1 = api_create_claim(data: data, quote: 'claim 1') - pm2 = api_create_claim(data: data, quote: 'claim 2') - api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) - @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" - wait_for_selector('#media-similarity__add-button') - wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click - @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element - wait_for_selector('.similarity-media-item__accept-relationship').click - @driver.navigate.refresh - wait_for_selector('.media__relationship') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # pin similar item - wait_for_selector('.media-similarity__menu-icon').click - wait_for_selector('.similarity-media-item__delete-relationship') - wait_for_selector('.similarity-media-item__pin-relationship').click - wait_for_url_change(@driver.current_url.to_s) - wait_for_selector('.media__relationship') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # remove similar item - wait_for_selector('.media-similarity__menu-icon').click - wait_for_selector('.similarity-media-item__pin-relationship') - wait_for_selector('.similarity-media-item__delete-relationship').click - wait_for_selector('.int-flash-message__toast') - @driver.navigate.refresh - wait_for_selector('.test__media') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 - end - - it 'should accept and reject suggested similarity', bin1: true do - data = api_create_team_and_bot - pm1 = api_create_claim(data: data, quote: 'claim 1') - pm2 = api_create_claim(data: data, quote: 'claim 2') - pm3 = api_create_claim(data: data, quote: 'claim 3') - api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) - api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm3.id) - @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" - wait_for_selector('#media-similarity__add-button') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 - wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click - @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element - wait_for_selector('.similarity-media-item__accept-relationship').click - @driver.action.move_to(wait_for_selector_list('.suggested-media__item')[1]).perform # hover element - wait_for_selector_list('.similarity-media-item__reject-relationship')[1].click - @driver.navigate.refresh - wait_for_selector('#media-similarity__add-button') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - end - + # it 'should import and export items', bin4: true do + # api_create_team_claims_sources_and_redirect_to_all_items({ count: 3 }) + # verbose_wait 2 # Wait for the items to be indexed in ElasticSearch + # wait_for_selector('.search__results-heading') + # wait_for_selector('.cluster-card').click + # wait_for_selector('#media-similarity__add-button').click + # + # # Merge similar items + # add_related_item('Claim 0') + # wait_for_selector('.media__relationship') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # expect(@driver.page_source.include?('Media')).to be(true) + # end + # + # it 'should pin and remove similarity items', bin4: true do + # data = api_create_team_and_bot + # pm1 = api_create_claim(data: data, quote: 'claim 1') + # pm2 = api_create_claim(data: data, quote: 'claim 2') + # api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) + # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" + # wait_for_selector('#media-similarity__add-button') + # wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click + # @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element + # wait_for_selector('.similarity-media-item__accept-relationship').click + # @driver.navigate.refresh + # wait_for_selector('.media__relationship') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # # pin similar item + # wait_for_selector('.media-similarity__menu-icon').click + # wait_for_selector('.similarity-media-item__delete-relationship') + # wait_for_selector('.similarity-media-item__pin-relationship').click + # wait_for_url_change(@driver.current_url.to_s) + # wait_for_selector('.media__relationship') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # # remove similar item + # wait_for_selector('.media-similarity__menu-icon').click + # wait_for_selector('.similarity-media-item__pin-relationship') + # wait_for_selector('.similarity-media-item__delete-relationship').click + # wait_for_selector('.int-flash-message__toast') + # @driver.navigate.refresh + # wait_for_selector('.test__media') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 + # end + # + # it 'should accept and reject suggested similarity', bin1: true do + # data = api_create_team_and_bot + # pm1 = api_create_claim(data: data, quote: 'claim 1') + # pm2 = api_create_claim(data: data, quote: 'claim 2') + # pm3 = api_create_claim(data: data, quote: 'claim 3') + # api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) + # api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm3.id) + # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" + # wait_for_selector('#media-similarity__add-button') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 + # wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click + # @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element + # wait_for_selector('.similarity-media-item__accept-relationship').click + # @driver.action.move_to(wait_for_selector_list('.suggested-media__item')[1]).perform # hover element + # wait_for_selector_list('.similarity-media-item__reject-relationship')[1].click + # @driver.navigate.refresh + # wait_for_selector('#media-similarity__add-button') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # end + # it 'should identify texts as similar', bin7: true do data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and TI industry. Lorem Ipsum has been used by the industry for text generation ever since the 1502s.') @@ -74,53 +74,53 @@ expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - it 'should identify videos as similar', bin7: true do - api_create_team_and_bot(bot: 'alegre') - @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - create_image('files/video.mp4') - verbose_wait 5 - wait_for_selector('.cluster-card') - create_image('files/video2.mp4') - verbose_wait 5 - wait_for_selector('.cluster-card').click - wait_for_selector('.media__more-medias') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - end - - it 'should identify images as similar', bin7: true do - api_create_team_and_bot(bot: 'alegre') - @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - create_image('files/similarity.jpg') - verbose_wait 4 - wait_for_selector('.cluster-card') - create_image('files/similarity2.jpg') - verbose_wait 4 - wait_for_selector('.cluster-card').click - wait_for_selector('.media__more-medias') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - end - - it 'should extract text from a image', bin7: true do - api_create_team_and_bot(bot: 'alegre') - @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - create_image('files/ocr.png') - verbose_wait 4 - wait_for_selector('.cluster-card').click - wait_for_selector('.image-media-card') - expect(@driver.page_source.include?('Extracted text')).to be(true) - expect(@driver.page_source.include?('Test')).to be(true) - end - - it 'should identify audios as similar', bin7: true do - api_create_team_and_bot(bot: 'alegre') - @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - create_image('files/audio.mp3') - verbose_wait 4 - wait_for_selector('.cluster-card') - create_image('files/audio.ogg') - verbose_wait 4 - wait_for_selector('.cluster-card').click - wait_for_selector('.media__more-medias') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - end + # it 'should identify videos as similar', bin7: true do + # api_create_team_and_bot(bot: 'alegre') + # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + # create_image('files/video.mp4') + # verbose_wait 5 + # wait_for_selector('.cluster-card') + # create_image('files/video2.mp4') + # verbose_wait 5 + # wait_for_selector('.cluster-card').click + # wait_for_selector('.media__more-medias') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # end + # + # it 'should identify images as similar', bin7: true do + # api_create_team_and_bot(bot: 'alegre') + # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + # create_image('files/similarity.jpg') + # verbose_wait 4 + # wait_for_selector('.cluster-card') + # create_image('files/similarity2.jpg') + # verbose_wait 4 + # wait_for_selector('.cluster-card').click + # wait_for_selector('.media__more-medias') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # end + # + # it 'should extract text from a image', bin7: true do + # api_create_team_and_bot(bot: 'alegre') + # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + # create_image('files/ocr.png') + # verbose_wait 4 + # wait_for_selector('.cluster-card').click + # wait_for_selector('.image-media-card') + # expect(@driver.page_source.include?('Extracted text')).to be(true) + # expect(@driver.page_source.include?('Test')).to be(true) + # end + # + # it 'should identify audios as similar', bin7: true do + # api_create_team_and_bot(bot: 'alegre') + # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + # create_image('files/audio.mp3') + # verbose_wait 4 + # wait_for_selector('.cluster-card') + # create_image('files/audio.ogg') + # verbose_wait 4 + # wait_for_selector('.cluster-card').click + # wait_for_selector('.media__more-medias') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # end end From dd6a203494323b57297cc9903060920cbabcfcdd Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 23 Sep 2024 22:14:55 -0300 Subject: [PATCH 16/71] Trying to make the test more robust --- test/spec/api_helpers.rb | 7 ++++++- test/spec/similarity_spec.rb | 8 ++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index 2affd07678..0a6ea1c820 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -7,7 +7,7 @@ def api_path "#{@config['api_path']}/test/" end - def request_api(path, params) + def request_api(path, params, attempts = 0) require 'net/http' uri = URI(api_path + path) uri.query = URI.encode_www_form(params) @@ -17,6 +17,11 @@ def request_api(path, params) ret = OpenStruct.new JSON.parse(response.body)['data'] rescue StandardError print "Failed to parse body of response for endpoint `#{path}`:\n#{response.inspect}\n" unless response.class <= Net::HTTPSuccess + rescue Net::ReadTimeout + if attempts < 5 + print "Timeout when calling #{path} with params '#{params.inspect}', retrying..." + request_api path, params, attempts + 1 + end end ret end diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 5811ab6573..10fbbad1ad 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,10 +65,10 @@ # it 'should identify texts as similar', bin7: true do data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and TI industry. Lorem Ipsum has been used by the industry for text generation ever since the 1502s.') - sleep 60 # wait for the items to be indexed in the Elasticsearch - api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and TI industry. Lorem Ipsum has been used by the industry for text generation ever since the 1501s.') - sleep 60 # wait for the items to be indexed in the Elasticsearch + pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + verbose_wait 5 + api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + verbose_wait 5 @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 From 33c6fe868bffdc587f0d061e2d6317cda7035388 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 23 Sep 2024 22:26:12 -0300 Subject: [PATCH 17/71] More robust test, take 2 --- test/spec/api_helpers.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index 0a6ea1c820..f907bd5c4c 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -15,13 +15,13 @@ def request_api(path, params, attempts = 0) ret = nil begin ret = OpenStruct.new JSON.parse(response.body)['data'] - rescue StandardError - print "Failed to parse body of response for endpoint `#{path}`:\n#{response.inspect}\n" unless response.class <= Net::HTTPSuccess rescue Net::ReadTimeout if attempts < 5 print "Timeout when calling #{path} with params '#{params.inspect}', retrying..." - request_api path, params, attempts + 1 + ret = request_api path, params, attempts + 1 end + rescue StandardError + print "Failed to parse body of response for endpoint `#{path}`:\n#{response.inspect}\n" unless response.class <= Net::HTTPSuccess end ret end From 042dbe7f99f7dab927dc6650905f43a36847d5b2 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 23 Sep 2024 22:41:14 -0300 Subject: [PATCH 18/71] [similarity tests] From e26a6b646c3125bcee2f3c1b1ba299ad77aca4ee Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 23 Sep 2024 23:34:40 -0300 Subject: [PATCH 19/71] [similarity tests] From 46b2428c927d2869e9f43efcb453c39715036bd4 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 23 Sep 2024 23:34:49 -0300 Subject: [PATCH 20/71] [similarity tests] --- test/spec/similarity_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 10fbbad1ad..7eab5e261e 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,10 +65,11 @@ # it 'should identify texts as similar', bin7: true do data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + verbose_wait 3 pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') - verbose_wait 5 + verbose_wait 10 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - verbose_wait 5 + verbose_wait 10 @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 From 5a9d8ac866e43e12be377f3e7cb20829b222ad4b Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:32:59 -0300 Subject: [PATCH 21/71] [similarity tests] --- test/spec/api_helpers.rb | 5 +++-- test/spec/similarity_spec.rb | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index f907bd5c4c..d66abbd171 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -17,8 +17,9 @@ def request_api(path, params, attempts = 0) ret = OpenStruct.new JSON.parse(response.body)['data'] rescue Net::ReadTimeout if attempts < 5 - print "Timeout when calling #{path} with params '#{params.inspect}', retrying..." - ret = request_api path, params, attempts + 1 + print "Timeout when calling #{path} with params '#{params.inspect}', retrying in 3 seconds..." + sleep 3 + ret = request_api(path, params, attempts + 1) end rescue StandardError print "Failed to parse body of response for endpoint `#{path}`:\n#{response.inspect}\n" unless response.class <= Net::HTTPSuccess diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 7eab5e261e..6d9ccda98e 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,7 +65,7 @@ # it 'should identify texts as similar', bin7: true do data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - verbose_wait 3 + verbose_wait 10 pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') verbose_wait 10 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') From e469a22c17205d96fcbf938b350c1da78e63b644 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:11:19 -0300 Subject: [PATCH 22/71] [similarity tests] --- test/spec/api_helpers.rb | 1 - test/spec/similarity_spec.rb | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index d66abbd171..e7f5c482c5 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -18,7 +18,6 @@ def request_api(path, params, attempts = 0) rescue Net::ReadTimeout if attempts < 5 print "Timeout when calling #{path} with params '#{params.inspect}', retrying in 3 seconds..." - sleep 3 ret = request_api(path, params, attempts + 1) end rescue StandardError diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 6d9ccda98e..7949748dba 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -64,8 +64,9 @@ # end # it 'should identify texts as similar', bin7: true do + puts 'Running test similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - verbose_wait 10 + verbose_wait 5 pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') verbose_wait 10 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') From f59e2b8006e2952022ede2fed1e93469f99ccaf9 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:55:18 -0300 Subject: [PATCH 23/71] Commenting Presto containers just to see if it makes any difference [similarity tests] --- docker-compose.yml | 130 ++++++++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f1b18b2a0e..ff00353e61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,67 +143,67 @@ services: SERVER_PORT: 3200 networks: - dev - presto-server: - build: presto - platform: linux/amd64 - ports: - - "8000:8000" - volumes: - - "./presto:/app" - env_file: - - presto/.env_file - environment: - ROLE: server - networks: - - dev - depends_on: - elasticmq: - condition: service_healthy - presto-image: - build: presto - platform: linux/amd64 - volumes: - - "./presto:/app" - env_file: - - presto/.env_file - environment: - ROLE: worker - MODEL_NAME: image.Model - networks: - - dev - depends_on: - elasticmq: - condition: service_healthy - presto-audio: - build: presto - platform: linux/amd64 - volumes: - - "./presto:/app" - env_file: - - presto/.env_file - environment: - ROLE: worker - MODEL_NAME: audio.Model - networks: - - dev - depends_on: - elasticmq: - condition: service_healthy - presto-video: - build: presto - platform: linux/amd64 - volumes: - - "./presto:/app" - env_file: - - presto/.env_file - environment: - ROLE: worker - MODEL_NAME: video.Model - networks: - - dev - depends_on: - elasticmq: - condition: service_healthy +# presto-server: +# build: presto +# platform: linux/amd64 +# ports: +# - "8000:8000" +# volumes: +# - "./presto:/app" +# env_file: +# - presto/.env_file +# environment: +# ROLE: server +# networks: +# - dev +# depends_on: +# elasticmq: +# condition: service_healthy +# presto-image: +# build: presto +# platform: linux/amd64 +# volumes: +# - "./presto:/app" +# env_file: +# - presto/.env_file +# environment: +# ROLE: worker +# MODEL_NAME: image.Model +# networks: +# - dev +# depends_on: +# elasticmq: +# condition: service_healthy +# presto-audio: +# build: presto +# platform: linux/amd64 +# volumes: +# - "./presto:/app" +# env_file: +# - presto/.env_file +# environment: +# ROLE: worker +# MODEL_NAME: audio.Model +# networks: +# - dev +# depends_on: +# elasticmq: +# condition: service_healthy +# presto-video: +# build: presto +# platform: linux/amd64 +# volumes: +# - "./presto:/app" +# env_file: +# - presto/.env_file +# environment: +# ROLE: worker +# MODEL_NAME: video.Model +# networks: +# - dev +# depends_on: +# elasticmq: +# condition: service_healthy alegre: build: alegre ports: @@ -213,10 +213,10 @@ services: depends_on: - redis - elasticsearch - - presto-server - - presto-image - - presto-audio - - presto-video + # - presto-server + # - presto-image + # - presto-audio + # - presto-video env_file: - alegre/.env_file networks: From a2940a5d7b466f7d6b74ffd15283d4f66b0f601f Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:57:44 -0300 Subject: [PATCH 24/71] Commenting Presto containers just to see if it makes any difference, take 2 [similarity tests] --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 035d92adc4..e471043acd 100755 --- a/build.sh +++ b/build.sh @@ -43,7 +43,7 @@ else docker-compose build docker-compose -f docker-compose.yml -f docker-test.yml up -d until curl --silent -I -f --fail http://localhost:3100; do printf .; sleep 1; done - until curl --silent -I -f --fail http://localhost:8000/ping; do printf .; sleep 1; done + # until curl --silent -I -f --fail http://localhost:8000/ping; do printf .; sleep 1; done fi until curl --silent -I -f --fail http://localhost:3200; do printf .; sleep 1; done until curl --silent -I -f --fail http://localhost:3000; do printf .; sleep 1; done From 0aa2987a85226405a5a00d8d73aa5587eb8db902 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 20:59:10 -0300 Subject: [PATCH 25/71] Commenting Presto containers just to see if it makes any difference, take 3 [similarity tests] --- docker-test.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docker-test.yml b/docker-test.yml index 4d0e04be16..62b7f3f252 100644 --- a/docker-test.yml +++ b/docker-test.yml @@ -41,19 +41,19 @@ services: environment: BOILERPLATE_ENV: test ALEGRE_PORT: 3100 - presto-server: - environment: - BOILERPLATE_ENV: test - PRESTO_PORT: 8000 - presto-audio: - environment: - BOILERPLATE_ENV: test - presto-image: - environment: - BOILERPLATE_ENV: test - presto-video: - environment: - BOILERPLATE_ENV: test +# presto-server: +# environment: +# BOILERPLATE_ENV: test +# PRESTO_PORT: 8000 +# presto-audio: +# environment: +# BOILERPLATE_ENV: test +# presto-image: +# environment: +# BOILERPLATE_ENV: test +# presto-video: +# environment: +# BOILERPLATE_ENV: test web: environment: PLATFORM: web From 3d0e2de02514dbab2596dcc6270b61f49d4c6051 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:03:17 -0300 Subject: [PATCH 26/71] Commenting Presto containers just to see if it makes any difference, take 4 [similarity tests] --- nginx.conf | 6 +++++- test/spec/api_helpers.rb | 2 +- test/spec/similarity_spec.rb | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/nginx.conf b/nginx.conf index 9bc91a8615..6debae62b9 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,4 +1,4 @@ -worker_processes 1; +worker_processes 3; events { worker_connections 50000; } @@ -24,4 +24,8 @@ http { } } + proxy_connect_timeout 30000; + proxy_send_timeout 30000; + proxy_read_timeout 30000; + send_timeout 30000; } diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index e7f5c482c5..1d527ec629 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -17,7 +17,7 @@ def request_api(path, params, attempts = 0) ret = OpenStruct.new JSON.parse(response.body)['data'] rescue Net::ReadTimeout if attempts < 5 - print "Timeout when calling #{path} with params '#{params.inspect}', retrying in 3 seconds..." + print "Timeout when calling #{path} with params '#{params.inspect}', retrying..." ret = request_api(path, params, attempts + 1) end rescue StandardError diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 7949748dba..793cffba0d 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -64,7 +64,7 @@ # end # it 'should identify texts as similar', bin7: true do - puts 'Running test similarity test...' + puts 'Running text similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) verbose_wait 5 pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') From 29b0ed244d314453f8abd4f208658e48dd2e0750 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:52:51 -0300 Subject: [PATCH 27/71] Commenting Presto containers just to see if it makes any difference, take 5 [similarity tests] --- test/spec/api_helpers.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index 1d527ec629..e86f4ffab1 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -7,7 +7,7 @@ def api_path "#{@config['api_path']}/test/" end - def request_api(path, params, attempts = 0) + def request_api(path, params) require 'net/http' uri = URI(api_path + path) uri.query = URI.encode_www_form(params) @@ -16,12 +16,9 @@ def request_api(path, params, attempts = 0) begin ret = OpenStruct.new JSON.parse(response.body)['data'] rescue Net::ReadTimeout - if attempts < 5 - print "Timeout when calling #{path} with params '#{params.inspect}', retrying..." - ret = request_api(path, params, attempts + 1) - end + print "Timeout when calling #{path} with params '#{params.inspect}'" rescue StandardError - print "Failed to parse body of response for endpoint `#{path}`:\n#{response.inspect}\n" unless response.class <= Net::HTTPSuccess + print "Failed to parse body of response for endpoint #{path}: Response: #{response.inspect} Body: #{response.body}" unless response.class <= Net::HTTPSuccess end ret end From 93d1ab0865de77866adb0c16e7215dde583d93bd Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:34:18 -0300 Subject: [PATCH 28/71] Commenting Presto containers just to see if it makes any difference, take 6 [similarity tests] --- nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx.conf b/nginx.conf index 6debae62b9..be8edc642f 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,4 +1,4 @@ -worker_processes 3; +worker_processes 1; events { worker_connections 50000; } From 0d88a0af9ee41bfeb760972d29b9f225fcbe9b6c Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:36:16 -0300 Subject: [PATCH 29/71] Not even creating an item, just to see if there is any Alegre activity [similarity tests] --- test/spec/similarity_spec.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 793cffba0d..d282ffa992 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -67,13 +67,13 @@ puts 'Running text similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) verbose_wait 5 - pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') - verbose_wait 10 - api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - verbose_wait 10 - @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" - wait_for_selector('.media__more-medias') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + # verbose_wait 10 + # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + # verbose_wait 10 + # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" + # wait_for_selector('.media__more-medias') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end # it 'should identify videos as similar', bin7: true do From 34321f4a2646bde7605c258dd0a23c65affc5038 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:58:42 -0300 Subject: [PATCH 30/71] Fixing lint issue [similarity tests] --- test/spec/similarity_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index d282ffa992..c65a59243a 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,7 +65,7 @@ # it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' - data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) verbose_wait 5 # pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') # verbose_wait 10 From ea3c62597d716149bfc5fa5f1aab7ae23e098841 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:27:09 -0300 Subject: [PATCH 31/71] Debugging claim creation [similarity tests] --- test/spec/similarity_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index c65a59243a..4b66780d13 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,9 +65,10 @@ # it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' - api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) verbose_wait 5 - # pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + puts "pm1 = #{pm1.inspect}" # verbose_wait 10 # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') # verbose_wait 10 From 9934bc19a8f0fc9ae43f4d923abea57a4fb9dce5 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 12:22:51 -0300 Subject: [PATCH 32/71] Rewriting in a way that we can set custom timeout values [similarity tests] --- test/spec/api_helpers.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index e86f4ffab1..aac9c3e133 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -11,9 +11,14 @@ def request_api(path, params) require 'net/http' uri = URI(api_path + path) uri.query = URI.encode_www_form(params) - response = Net::HTTP.get_response(uri) ret = nil begin + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = (uri.scheme == 'https') + http.open_timeout = 300 # Time to open the connection + http.read_timeout = 300 # Time to wait for the response + request = Net::HTTP::Get.new(uri.request_uri) + response = http.request(request) ret = OpenStruct.new JSON.parse(response.body)['data'] rescue Net::ReadTimeout print "Timeout when calling #{path} with params '#{params.inspect}'" From e95c762b09fa7bc5c9916a831fb59a46dcb7e6ce Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:51:51 -0300 Subject: [PATCH 33/71] Enabling Check API logs as well [similarity tests] --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index e471043acd..db1994c522 100755 --- a/build.sh +++ b/build.sh @@ -49,7 +49,7 @@ else until curl --silent -I -f --fail http://localhost:3000; do printf .; sleep 1; done # Uncomment to debug Check API and Alegre. Warning: This can lead to Travis error "The job exceeded the maximum log length, and has been terminated.". # tail -f check-api/log/test.log & - # docker-compose logs -f api & + docker-compose logs -f api & docker-compose logs -f alegre & # docker-compose logs -f presto-server & # docker-compose logs -f presto-image & From 253bd212e46a873a1a021edc07806b8870dbb35f Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:49:29 -0300 Subject: [PATCH 34/71] Enabling Check API logs as well, take 2 [similarity tests] --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index db1994c522..73eeb16156 100755 --- a/build.sh +++ b/build.sh @@ -48,7 +48,7 @@ else until curl --silent -I -f --fail http://localhost:3200; do printf .; sleep 1; done until curl --silent -I -f --fail http://localhost:3000; do printf .; sleep 1; done # Uncomment to debug Check API and Alegre. Warning: This can lead to Travis error "The job exceeded the maximum log length, and has been terminated.". - # tail -f check-api/log/test.log & + tail -f check-api/log/test.log & docker-compose logs -f api & docker-compose logs -f alegre & # docker-compose logs -f presto-server & From 93374902c8cacdc8744a1aada787cccf325b667f Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 18:03:07 -0300 Subject: [PATCH 35/71] [similarity tests] From 69ae18acfb08361c5294428b67087b847a399e92 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:20:48 -0300 Subject: [PATCH 36/71] Time to run the test! [similarity tests] --- test/spec/similarity_spec.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 4b66780d13..793cffba0d 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -68,13 +68,12 @@ data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) verbose_wait 5 pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') - puts "pm1 = #{pm1.inspect}" - # verbose_wait 10 - # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - # verbose_wait 10 - # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" - # wait_for_selector('.media__more-medias') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + verbose_wait 10 + api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + verbose_wait 10 + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end # it 'should identify videos as similar', bin7: true do From 066d69cb5c849edd671b1eb9d45f83c5bc3e3dd9 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 25 Sep 2024 22:38:06 -0300 Subject: [PATCH 37/71] Time to run the test! Take 2! [similarity tests] --- build.sh | 6 +++--- test/spec/api_helpers.rb | 6 ++++-- test/spec/similarity_spec.rb | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 73eeb16156..ca0be2d5b1 100755 --- a/build.sh +++ b/build.sh @@ -48,9 +48,9 @@ else until curl --silent -I -f --fail http://localhost:3200; do printf .; sleep 1; done until curl --silent -I -f --fail http://localhost:3000; do printf .; sleep 1; done # Uncomment to debug Check API and Alegre. Warning: This can lead to Travis error "The job exceeded the maximum log length, and has been terminated.". - tail -f check-api/log/test.log & - docker-compose logs -f api & - docker-compose logs -f alegre & + # tail -f check-api/log/test.log & + # docker-compose logs -f api & + # docker-compose logs -f alegre & # docker-compose logs -f presto-server & # docker-compose logs -f presto-image & # docker-compose logs -f presto-audio & diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index aac9c3e133..609330a8d6 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -12,6 +12,7 @@ def request_api(path, params) uri = URI(api_path + path) uri.query = URI.encode_www_form(params) ret = nil + response = nil begin http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == 'https') @@ -20,10 +21,11 @@ def request_api(path, params) request = Net::HTTP::Get.new(uri.request_uri) response = http.request(request) ret = OpenStruct.new JSON.parse(response.body)['data'] + puts "Successful response when calling #{path} with params '#{params.inspect}': #{response.body}" rescue Net::ReadTimeout - print "Timeout when calling #{path} with params '#{params.inspect}'" + puts "Timeout when calling #{path} with params '#{params.inspect}'" rescue StandardError - print "Failed to parse body of response for endpoint #{path}: Response: #{response.inspect} Body: #{response.body}" unless response.class <= Net::HTTPSuccess + puts "Failed to parse body of response for endpoint #{path}: Response: #{response.inspect} Body: #{response.body}" unless response.class <= Net::HTTPSuccess end ret end diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 793cffba0d..9b78ffa533 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -66,12 +66,12 @@ it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - verbose_wait 5 - pm1 = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}" + pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') verbose_wait 10 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') verbose_wait 10 - @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end From 83e691e590c6c210b609c5b396cad63a60fc298e Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:32:50 -0300 Subject: [PATCH 38/71] Time to run the test! Take 3! [similarity tests] --- nginx.conf | 5 ----- test/spec/similarity_spec.rb | 5 ++++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nginx.conf b/nginx.conf index be8edc642f..e6a5d7fdf8 100644 --- a/nginx.conf +++ b/nginx.conf @@ -23,9 +23,4 @@ http { proxy_set_header X-Forwarded-Host $server_name; } } - - proxy_connect_timeout 30000; - proxy_send_timeout 30000; - proxy_read_timeout 30000; - send_timeout 30000; } diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 9b78ffa533..0bbf602459 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -66,11 +66,14 @@ it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}" + puts 'Created team and bot...' pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + puts 'Created first claim...' verbose_wait 10 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + puts 'Created second claim...' verbose_wait 10 + puts 'Opening page...' @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 From 77408dea0135652f51d9dbd31c9d2f1d0cfa1c64 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:05:02 -0300 Subject: [PATCH 39/71] Debug... [similarity tests] --- test/spec/similarity_spec.rb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 0bbf602459..b2683b5201 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,18 +65,18 @@ # it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' - data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - puts 'Created team and bot...' - pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') - puts 'Created first claim...' - verbose_wait 10 - api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - puts 'Created second claim...' - verbose_wait 10 - puts 'Opening page...' - @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" - wait_for_selector('.media__more-medias') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + # puts 'Created team and bot...' + # pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + # puts 'Created first claim...' + # verbose_wait 10 + # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + # puts 'Created second claim...' + # verbose_wait 10 + # puts 'Opening page...' + # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + # wait_for_selector('.media__more-medias') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end # it 'should identify videos as similar', bin7: true do From 03b12d46eb9cc83552daf312d2d1cd45e425d8d6 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:25:54 -0300 Subject: [PATCH 40/71] Debug 2... [similarity tests] --- test/spec/similarity_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index b2683b5201..35839e2868 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,8 +65,8 @@ # it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' - # data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - # puts 'Created team and bot...' + data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + puts 'Created team and bot...' # pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') # puts 'Created first claim...' # verbose_wait 10 From 4649dab4778c389f84394a322d7b6bf3c8944891 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:52:28 -0300 Subject: [PATCH 41/71] Debug 3... [similarity tests] --- test/spec/similarity_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 35839e2868..009ad63446 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,7 +65,7 @@ # it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' - data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) puts 'Created team and bot...' # pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') # puts 'Created first claim...' From 483a9e2bb092c2903c6f0822f54a73a37d703a73 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:18:40 -0300 Subject: [PATCH 42/71] Debug 4... [similarity tests] --- test/spec/api_helpers.rb | 4 ++-- test/spec/similarity_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index 609330a8d6..f60def77b6 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -16,8 +16,8 @@ def request_api(path, params) begin http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == 'https') - http.open_timeout = 300 # Time to open the connection - http.read_timeout = 300 # Time to wait for the response + http.open_timeout = 120 # Time to open the connection + http.read_timeout = 120 # Time to wait for the response request = Net::HTTP::Get.new(uri.request_uri) response = http.request(request) ret = OpenStruct.new JSON.parse(response.body)['data'] diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 009ad63446..fe9f28ac35 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -65,9 +65,9 @@ # it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' - api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) puts 'Created team and bot...' - # pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') # puts 'Created first claim...' # verbose_wait 10 # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') From c941f672e9c21724ccb5c272a6138fefb6ccb9e2 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 26 Sep 2024 23:04:41 -0300 Subject: [PATCH 43/71] Debug 5... [similarity tests] --- test/spec/similarity_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index fe9f28ac35..622e937e66 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -68,7 +68,7 @@ data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) puts 'Created team and bot...' api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') - # puts 'Created first claim...' + puts 'Created first claim...' # verbose_wait 10 # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') # puts 'Created second claim...' From 6992b7fb8f2edc100ab1dc68f0d00d49911be578 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:16:22 -0300 Subject: [PATCH 44/71] Debug 6... [similarity tests] From 95062c2d4541caba1a87386c92e97518d460e1a5 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:23:18 -0300 Subject: [PATCH 45/71] Debug 6... [similarity tests] From 77e45f2859f1c195091fca4441bc51bf215432ae Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:58:10 -0300 Subject: [PATCH 46/71] Debug 7... [similarity tests] From c842ac418f065dfb621300429edd9cfd0f457393 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:20:52 -0300 Subject: [PATCH 47/71] Debug 8... [similarity tests] From 712d45c10c5f672afddf377b557407f7857d4aaf Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:43:20 -0300 Subject: [PATCH 48/71] Debug 9... [similarity tests] --- test/spec/similarity_spec.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 622e937e66..8b47ef5c80 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -69,12 +69,12 @@ puts 'Created team and bot...' api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' - # verbose_wait 10 - # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - # puts 'Created second claim...' - # verbose_wait 10 - # puts 'Opening page...' - # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + verbose_wait 5 + api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + puts 'Created second claim...' + verbose_wait 5 + puts 'Opening page...' + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" # wait_for_selector('.media__more-medias') # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end From 96b37c9e2990f7d9cf9cec73e9425a0135bd5543 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:43:47 -0300 Subject: [PATCH 49/71] Debug 9... [similarity tests] --- test/spec/similarity_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 8b47ef5c80..5fa3f933b2 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -73,8 +73,8 @@ api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') puts 'Created second claim...' verbose_wait 5 - puts 'Opening page...' - @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + # puts 'Opening page...' + # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" # wait_for_selector('.media__more-medias') # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end From 1625047bd2b7767177455917e6a328c85b86fb22 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:18:56 -0300 Subject: [PATCH 50/71] Debug 10... [similarity tests] --- test/spec/similarity_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 5fa3f933b2..761176d2f7 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -69,10 +69,10 @@ puts 'Created team and bot...' api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' - verbose_wait 5 - api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - puts 'Created second claim...' - verbose_wait 5 + verbose_wait 10 + # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + # puts 'Created second claim...' + # verbose_wait 5 # puts 'Opening page...' # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" # wait_for_selector('.media__more-medias') From 68d983d310e706652b915ba3625f6db030b7e20a Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:53:01 -0300 Subject: [PATCH 51/71] Debug 11... [similarity tests] --- test/spec/similarity_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 761176d2f7..11acd48123 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -69,7 +69,7 @@ puts 'Created team and bot...' api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' - verbose_wait 10 + verbose_wait 5 # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') # puts 'Created second claim...' # verbose_wait 5 From 8613ebc6cc56c02176dad926e35d023d897f746a Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:28:55 -0300 Subject: [PATCH 52/71] Debug 12... [similarity tests] --- test/spec/similarity_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 11acd48123..5fa3f933b2 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -70,9 +70,9 @@ api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' verbose_wait 5 - # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - # puts 'Created second claim...' - # verbose_wait 5 + api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + puts 'Created second claim...' + verbose_wait 5 # puts 'Opening page...' # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" # wait_for_selector('.media__more-medias') From b23170d335d35256061d6bba5b6280fa7dbfbbce Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:49:42 -0300 Subject: [PATCH 53/71] Debug 13... [similarity tests] --- test/spec/similarity_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 5fa3f933b2..62e2b00ee7 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -69,10 +69,10 @@ puts 'Created team and bot...' api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' - verbose_wait 5 + verbose_wait 4 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') puts 'Created second claim...' - verbose_wait 5 + verbose_wait 4 # puts 'Opening page...' # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" # wait_for_selector('.media__more-medias') From 87c62a8916926565a5f12a9100d6b0868751dd19 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:21:55 -0300 Subject: [PATCH 54/71] Debug 14... [similarity tests] --- test/spec/similarity_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 62e2b00ee7..f3cc6a8832 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -69,10 +69,10 @@ puts 'Created team and bot...' api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' - verbose_wait 4 + verbose_wait 3 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') puts 'Created second claim...' - verbose_wait 4 + verbose_wait 3 # puts 'Opening page...' # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" # wait_for_selector('.media__more-medias') From ae2a2b92a36a99125f1c2801dfd9dce09a0710ff Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Sep 2024 22:01:14 -0300 Subject: [PATCH 55/71] Debug 15... [similarity tests] --- test/spec/similarity_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index f3cc6a8832..2b586b19a0 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -73,8 +73,9 @@ api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') puts 'Created second claim...' verbose_wait 3 - # puts 'Opening page...' - # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + puts 'Opening page...' + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + puts 'Page opened!' # wait_for_selector('.media__more-medias') # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end From e32eb9dc35c6699e3dd6ddfb37e7ce1e4c879d05 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sat, 28 Sep 2024 00:46:25 -0300 Subject: [PATCH 56/71] Debug 16... [similarity tests] --- test/spec/similarity_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 2b586b19a0..cb504a7579 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -70,7 +70,7 @@ api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' verbose_wait 3 - api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') puts 'Created second claim...' verbose_wait 3 puts 'Opening page...' From a8427acfc811ca1b390bf05270670a6655ca32d7 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sat, 28 Sep 2024 11:12:47 -0300 Subject: [PATCH 57/71] Debug 17... [similarity tests] --- test/spec/similarity_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index cb504a7579..948d544f87 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -76,8 +76,8 @@ puts 'Opening page...' @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" puts 'Page opened!' - # wait_for_selector('.media__more-medias') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end # it 'should identify videos as similar', bin7: true do From 6202bd93ff5eaf5b4a2f0fc16c91fd225a2eb2c8 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sat, 28 Sep 2024 17:28:57 -0300 Subject: [PATCH 58/71] Fixing, take 1... [similarity tests] --- test/spec/similarity_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 948d544f87..fde9dd3f44 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -67,10 +67,10 @@ puts 'Running text similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) puts 'Created team and bot...' - api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') puts 'Created first claim...' verbose_wait 3 - pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') puts 'Created second claim...' verbose_wait 3 puts 'Opening page...' From 0b44b6dd7be14fae592b9b309e0f04bf07052bcf Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sat, 28 Sep 2024 21:54:54 -0300 Subject: [PATCH 59/71] Run all tests and run Presto [similarity tests] --- build.sh | 2 +- docker-compose.yml | 130 ++++++++++---------- docker-test.yml | 26 ++-- test/spec/similarity_spec.rb | 226 +++++++++++++++++------------------ 4 files changed, 192 insertions(+), 192 deletions(-) diff --git a/build.sh b/build.sh index ca0be2d5b1..8ae56ac434 100755 --- a/build.sh +++ b/build.sh @@ -43,7 +43,7 @@ else docker-compose build docker-compose -f docker-compose.yml -f docker-test.yml up -d until curl --silent -I -f --fail http://localhost:3100; do printf .; sleep 1; done - # until curl --silent -I -f --fail http://localhost:8000/ping; do printf .; sleep 1; done + until curl --silent -I -f --fail http://localhost:8000/ping; do printf .; sleep 1; done fi until curl --silent -I -f --fail http://localhost:3200; do printf .; sleep 1; done until curl --silent -I -f --fail http://localhost:3000; do printf .; sleep 1; done diff --git a/docker-compose.yml b/docker-compose.yml index ff00353e61..f1b18b2a0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,67 +143,67 @@ services: SERVER_PORT: 3200 networks: - dev -# presto-server: -# build: presto -# platform: linux/amd64 -# ports: -# - "8000:8000" -# volumes: -# - "./presto:/app" -# env_file: -# - presto/.env_file -# environment: -# ROLE: server -# networks: -# - dev -# depends_on: -# elasticmq: -# condition: service_healthy -# presto-image: -# build: presto -# platform: linux/amd64 -# volumes: -# - "./presto:/app" -# env_file: -# - presto/.env_file -# environment: -# ROLE: worker -# MODEL_NAME: image.Model -# networks: -# - dev -# depends_on: -# elasticmq: -# condition: service_healthy -# presto-audio: -# build: presto -# platform: linux/amd64 -# volumes: -# - "./presto:/app" -# env_file: -# - presto/.env_file -# environment: -# ROLE: worker -# MODEL_NAME: audio.Model -# networks: -# - dev -# depends_on: -# elasticmq: -# condition: service_healthy -# presto-video: -# build: presto -# platform: linux/amd64 -# volumes: -# - "./presto:/app" -# env_file: -# - presto/.env_file -# environment: -# ROLE: worker -# MODEL_NAME: video.Model -# networks: -# - dev -# depends_on: -# elasticmq: -# condition: service_healthy + presto-server: + build: presto + platform: linux/amd64 + ports: + - "8000:8000" + volumes: + - "./presto:/app" + env_file: + - presto/.env_file + environment: + ROLE: server + networks: + - dev + depends_on: + elasticmq: + condition: service_healthy + presto-image: + build: presto + platform: linux/amd64 + volumes: + - "./presto:/app" + env_file: + - presto/.env_file + environment: + ROLE: worker + MODEL_NAME: image.Model + networks: + - dev + depends_on: + elasticmq: + condition: service_healthy + presto-audio: + build: presto + platform: linux/amd64 + volumes: + - "./presto:/app" + env_file: + - presto/.env_file + environment: + ROLE: worker + MODEL_NAME: audio.Model + networks: + - dev + depends_on: + elasticmq: + condition: service_healthy + presto-video: + build: presto + platform: linux/amd64 + volumes: + - "./presto:/app" + env_file: + - presto/.env_file + environment: + ROLE: worker + MODEL_NAME: video.Model + networks: + - dev + depends_on: + elasticmq: + condition: service_healthy alegre: build: alegre ports: @@ -213,10 +213,10 @@ services: depends_on: - redis - elasticsearch - # - presto-server - # - presto-image - # - presto-audio - # - presto-video + - presto-server + - presto-image + - presto-audio + - presto-video env_file: - alegre/.env_file networks: diff --git a/docker-test.yml b/docker-test.yml index 62b7f3f252..4d0e04be16 100644 --- a/docker-test.yml +++ b/docker-test.yml @@ -41,19 +41,19 @@ services: environment: BOILERPLATE_ENV: test ALEGRE_PORT: 3100 -# presto-server: -# environment: -# BOILERPLATE_ENV: test -# PRESTO_PORT: 8000 -# presto-audio: -# environment: -# BOILERPLATE_ENV: test -# presto-image: -# environment: -# BOILERPLATE_ENV: test -# presto-video: -# environment: -# BOILERPLATE_ENV: test + presto-server: + environment: + BOILERPLATE_ENV: test + PRESTO_PORT: 8000 + presto-audio: + environment: + BOILERPLATE_ENV: test + presto-image: + environment: + BOILERPLATE_ENV: test + presto-video: + environment: + BOILERPLATE_ENV: test web: environment: PLATFORM: web diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index fde9dd3f44..85ea6cb977 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -1,68 +1,68 @@ shared_examples 'similarity' do - # it 'should import and export items', bin4: true do - # api_create_team_claims_sources_and_redirect_to_all_items({ count: 3 }) - # verbose_wait 2 # Wait for the items to be indexed in ElasticSearch - # wait_for_selector('.search__results-heading') - # wait_for_selector('.cluster-card').click - # wait_for_selector('#media-similarity__add-button').click - # - # # Merge similar items - # add_related_item('Claim 0') - # wait_for_selector('.media__relationship') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # expect(@driver.page_source.include?('Media')).to be(true) - # end - # - # it 'should pin and remove similarity items', bin4: true do - # data = api_create_team_and_bot - # pm1 = api_create_claim(data: data, quote: 'claim 1') - # pm2 = api_create_claim(data: data, quote: 'claim 2') - # api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) - # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" - # wait_for_selector('#media-similarity__add-button') - # wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click - # @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element - # wait_for_selector('.similarity-media-item__accept-relationship').click - # @driver.navigate.refresh - # wait_for_selector('.media__relationship') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # # pin similar item - # wait_for_selector('.media-similarity__menu-icon').click - # wait_for_selector('.similarity-media-item__delete-relationship') - # wait_for_selector('.similarity-media-item__pin-relationship').click - # wait_for_url_change(@driver.current_url.to_s) - # wait_for_selector('.media__relationship') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # # remove similar item - # wait_for_selector('.media-similarity__menu-icon').click - # wait_for_selector('.similarity-media-item__pin-relationship') - # wait_for_selector('.similarity-media-item__delete-relationship').click - # wait_for_selector('.int-flash-message__toast') - # @driver.navigate.refresh - # wait_for_selector('.test__media') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 - # end - # - # it 'should accept and reject suggested similarity', bin1: true do - # data = api_create_team_and_bot - # pm1 = api_create_claim(data: data, quote: 'claim 1') - # pm2 = api_create_claim(data: data, quote: 'claim 2') - # pm3 = api_create_claim(data: data, quote: 'claim 3') - # api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) - # api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm3.id) - # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" - # wait_for_selector('#media-similarity__add-button') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 - # wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click - # @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element - # wait_for_selector('.similarity-media-item__accept-relationship').click - # @driver.action.move_to(wait_for_selector_list('.suggested-media__item')[1]).perform # hover element - # wait_for_selector_list('.similarity-media-item__reject-relationship')[1].click - # @driver.navigate.refresh - # wait_for_selector('#media-similarity__add-button') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # end - # + it 'should import and export items', bin4: true do + api_create_team_claims_sources_and_redirect_to_all_items({ count: 3 }) + verbose_wait 2 # Wait for the items to be indexed in ElasticSearch + wait_for_selector('.search__results-heading') + wait_for_selector('.cluster-card').click + wait_for_selector('#media-similarity__add-button').click + + # Merge similar items + add_related_item('Claim 0') + wait_for_selector('.media__relationship') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + expect(@driver.page_source.include?('Media')).to be(true) + end + + it 'should pin and remove similarity items', bin4: true do + data = api_create_team_and_bot + pm1 = api_create_claim(data: data, quote: 'claim 1') + pm2 = api_create_claim(data: data, quote: 'claim 2') + api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" + wait_for_selector('#media-similarity__add-button') + wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click + @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element + wait_for_selector('.similarity-media-item__accept-relationship').click + @driver.navigate.refresh + wait_for_selector('.media__relationship') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # pin similar item + wait_for_selector('.media-similarity__menu-icon').click + wait_for_selector('.similarity-media-item__delete-relationship') + wait_for_selector('.similarity-media-item__pin-relationship').click + wait_for_url_change(@driver.current_url.to_s) + wait_for_selector('.media__relationship') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # remove similar item + wait_for_selector('.media-similarity__menu-icon').click + wait_for_selector('.similarity-media-item__pin-relationship') + wait_for_selector('.similarity-media-item__delete-relationship').click + wait_for_selector('.int-flash-message__toast') + @driver.navigate.refresh + wait_for_selector('.test__media') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 + end + + it 'should accept and reject suggested similarity', bin1: true do + data = api_create_team_and_bot + pm1 = api_create_claim(data: data, quote: 'claim 1') + pm2 = api_create_claim(data: data, quote: 'claim 2') + pm3 = api_create_claim(data: data, quote: 'claim 3') + api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm2.id) + api_suggest_similarity_between_items(data[:team].dbid, pm1.id, pm3.id) + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm1.id}" + wait_for_selector('#media-similarity__add-button') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 + wait_for_selector("//span[contains(text(), 'Suggestions')]", :xpath).click + @driver.action.move_to(wait_for_selector('.suggested-media__item')).perform # hover element + wait_for_selector('.similarity-media-item__accept-relationship').click + @driver.action.move_to(wait_for_selector_list('.suggested-media__item')[1]).perform # hover element + wait_for_selector_list('.similarity-media-item__reject-relationship')[1].click + @driver.navigate.refresh + wait_for_selector('#media-similarity__add-button') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + end + it 'should identify texts as similar', bin7: true do puts 'Running text similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) @@ -80,53 +80,53 @@ expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - # it 'should identify videos as similar', bin7: true do - # api_create_team_and_bot(bot: 'alegre') - # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - # create_image('files/video.mp4') - # verbose_wait 5 - # wait_for_selector('.cluster-card') - # create_image('files/video2.mp4') - # verbose_wait 5 - # wait_for_selector('.cluster-card').click - # wait_for_selector('.media__more-medias') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # end - # - # it 'should identify images as similar', bin7: true do - # api_create_team_and_bot(bot: 'alegre') - # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - # create_image('files/similarity.jpg') - # verbose_wait 4 - # wait_for_selector('.cluster-card') - # create_image('files/similarity2.jpg') - # verbose_wait 4 - # wait_for_selector('.cluster-card').click - # wait_for_selector('.media__more-medias') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # end - # - # it 'should extract text from a image', bin7: true do - # api_create_team_and_bot(bot: 'alegre') - # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - # create_image('files/ocr.png') - # verbose_wait 4 - # wait_for_selector('.cluster-card').click - # wait_for_selector('.image-media-card') - # expect(@driver.page_source.include?('Extracted text')).to be(true) - # expect(@driver.page_source.include?('Test')).to be(true) - # end - # - # it 'should identify audios as similar', bin7: true do - # api_create_team_and_bot(bot: 'alegre') - # @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - # create_image('files/audio.mp3') - # verbose_wait 4 - # wait_for_selector('.cluster-card') - # create_image('files/audio.ogg') - # verbose_wait 4 - # wait_for_selector('.cluster-card').click - # wait_for_selector('.media__more-medias') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # end + it 'should identify videos as similar', bin7: true do + api_create_team_and_bot(bot: 'alegre') + @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + create_image('files/video.mp4') + verbose_wait 5 + wait_for_selector('.cluster-card') + create_image('files/video2.mp4') + verbose_wait 5 + wait_for_selector('.cluster-card').click + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + end + + it 'should identify images as similar', bin7: true do + api_create_team_and_bot(bot: 'alegre') + @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + create_image('files/similarity.jpg') + verbose_wait 4 + wait_for_selector('.cluster-card') + create_image('files/similarity2.jpg') + verbose_wait 4 + wait_for_selector('.cluster-card').click + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + end + + it 'should extract text from a image', bin7: true do + api_create_team_and_bot(bot: 'alegre') + @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + create_image('files/ocr.png') + verbose_wait 4 + wait_for_selector('.cluster-card').click + wait_for_selector('.image-media-card') + expect(@driver.page_source.include?('Extracted text')).to be(true) + expect(@driver.page_source.include?('Test')).to be(true) + end + + it 'should identify audios as similar', bin7: true do + api_create_team_and_bot(bot: 'alegre') + @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" + create_image('files/audio.mp3') + verbose_wait 4 + wait_for_selector('.cluster-card') + create_image('files/audio.ogg') + verbose_wait 4 + wait_for_selector('.cluster-card').click + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + end end From a55973fa271e40458416933b4f8c010491c6bc0d Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sat, 28 Sep 2024 21:56:04 -0300 Subject: [PATCH 60/71] Run all tests and run Presto, take 2 [similarity tests] --- test/spec/similarity_spec.rb | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 85ea6cb977..76e86dc9fa 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -5,14 +5,14 @@ wait_for_selector('.search__results-heading') wait_for_selector('.cluster-card').click wait_for_selector('#media-similarity__add-button').click - + # Merge similar items add_related_item('Claim 0') wait_for_selector('.media__relationship') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 expect(@driver.page_source.include?('Media')).to be(true) end - + it 'should pin and remove similarity items', bin4: true do data = api_create_team_and_bot pm1 = api_create_claim(data: data, quote: 'claim 1') @@ -42,7 +42,7 @@ wait_for_selector('.test__media') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 0 end - + it 'should accept and reject suggested similarity', bin1: true do data = api_create_team_and_bot pm1 = api_create_claim(data: data, quote: 'claim 1') @@ -62,20 +62,14 @@ wait_for_selector('#media-similarity__add-button') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - + it 'should identify texts as similar', bin7: true do - puts 'Running text similarity test...' data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) - puts 'Created team and bot...' pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') - puts 'Created first claim...' verbose_wait 3 api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - puts 'Created second claim...' verbose_wait 3 - puts 'Opening page...' @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" - puts 'Page opened!' wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end @@ -92,7 +86,7 @@ wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - + it 'should identify images as similar', bin7: true do api_create_team_and_bot(bot: 'alegre') @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" @@ -105,7 +99,7 @@ wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - + it 'should extract text from a image', bin7: true do api_create_team_and_bot(bot: 'alegre') @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" @@ -116,7 +110,7 @@ expect(@driver.page_source.include?('Extracted text')).to be(true) expect(@driver.page_source.include?('Test')).to be(true) end - + it 'should identify audios as similar', bin7: true do api_create_team_and_bot(bot: 'alegre') @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" From c15ef8ebf6436690723892447cb6c4605f9ac3cd Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Fri, 11 Oct 2024 20:02:07 -0700 Subject: [PATCH 61/71] Splitting CI between media and text Just a refactoring. Two separate builds in CI, one for text and another one for media. --------- Co-authored-by: danielevalverde Co-authored-by: Caio <117518+caiosba@users.noreply.github.com> --- .travis.yml | 22 +++++++- build.sh | 56 ++++++++++--------- docker-compose.yml | 18 +++++- docker-test.yml | 3 + test/parallel-media-similarity-test.sh | 22 ++++++++ ...st.sh => parallel-text-similarity-test.sh} | 0 test/run-tests.sh | 8 ++- test/spec/api_helpers.rb | 7 ++- test/spec/app_spec_helpers.rb | 2 +- test/spec/similarity_spec.rb | 49 +++++++++++----- 10 files changed, 137 insertions(+), 50 deletions(-) create mode 100755 test/parallel-media-similarity-test.sh rename test/{parallel-similarity-test.sh => parallel-text-similarity-test.sh} (100%) diff --git a/.travis.yml b/.travis.yml index 91a9d430ee..2d619232c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,8 +82,24 @@ jobs: -e TEST_RETRY_COUNT=3 web npm run tests - stage: tests - name: similarity-tests - if: branch = develop OR branch = master OR commit_message =~ /\[full ci\]/ OR commit_message =~ /\[similarity tests\]/ + name: media-similarity-tests + if: branch = develop OR branch = master OR commit_message =~ /\[full ci\]/ OR commit_message =~ /\[media similarity tests\]/ + script: + - docker-compose exec web service nginx start + - docker-compose -f docker-compose.yml -f docker-test.yml exec chromedriver service nginx start + - docker-compose exec + -e IMGUR_CLIENT_ID=$IMGUR_CLIENT_ID + -e TRAVIS_JOB_NAME=$TRAVIS_JOB_NAME + -e TRAVIS_BRANCH=$TRAVIS_BRANCH + -e AWS_ENDPOINT=$AWS_ENDPOINT + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + -e AWS_REGION=$AWS_REGION + -e TEST_RETRY_COUNT=3 + web npm run tests + - stage: tests + name: text-similarity-tests + if: branch = develop OR branch = master OR commit_message =~ /\[full ci\]/ OR commit_message =~ /\[text similarity tests\]/ script: - docker-compose exec web service nginx start - docker-compose -f docker-compose.yml -f docker-test.yml exec chromedriver service nginx start @@ -99,7 +115,7 @@ jobs: web npm run tests - stage: tests name: unit-tests - if: branch != develop AND branch != master AND NOT commit_message =~ /\[full ci\]/ AND NOT commit_message =~ /\[smoke tests\]/ AND NOT commit_message =~ /\[similarity tests\]/ + if: branch != develop AND branch != master AND NOT commit_message =~ /\[full ci\]/ AND NOT commit_message =~ /\[smoke tests\]/ AND NOT commit_message =~ /\[media similarity tests\]/ AND NOT commit_message =~ /\[text similarity tests\]/ script: - docker-compose exec -e TRAVIS_JOB_NAME=$TRAVIS_JOB_NAME web npm run tests after_script: diff --git a/build.sh b/build.sh index 8ae56ac434..67607b74d0 100755 --- a/build.sh +++ b/build.sh @@ -1,7 +1,7 @@ #!/bin/bash # Running only unit tests -if [[ $TRAVIS_BRANCH != 'develop' && $TRAVIS_BRANCH != 'master' && ! $TRAVIS_COMMIT_MESSAGE =~ \[full\ ci\] && ! $TRAVIS_COMMIT_MESSAGE =~ \[smoke\ tests\] && ! $TRAVIS_COMMIT_MESSAGE =~ \[similarity\ tests\] ]] +if [[ $TRAVIS_BRANCH != 'develop' && $TRAVIS_BRANCH != 'master' && ! $TRAVIS_COMMIT_MESSAGE =~ \[full\ ci\] && ! $TRAVIS_COMMIT_MESSAGE =~ \[smoke\ tests\] && ! $TRAVIS_COMMIT_MESSAGE =~ \[text\ similarity\ tests\] && ! $TRAVIS_COMMIT_MESSAGE =~ \[media\ similarity\ tests\] ]] then echo "Running only unit tests" docker-compose build web @@ -14,34 +14,40 @@ else docker-compose build web api api-background pender pender-background docker-compose -f docker-compose.yml -f docker-test.yml up -d web api api-background pender pender-background chromedriver else - i=0 - NGROK_URL="" - ngrok config add-authtoken $NGROK_AUTH - ngrok config upgrade 2 - while [ -z "$NGROK_URL" -a $i -lt 5 ]; do - i=$(($i + 1)) - ngrok http 9000 >/dev/null & - until curl --silent -I -f --fail http://localhost:4040; do printf "."; sleep 10; done - curl -I -v http://localhost:4040 - curl localhost:4040/api/tunnels > ngrok.json - cat ngrok.json - NGROK_URL=$(grep -Po '"public_url": *\K"[^"]*"' ngrok.json | tail -n1 | sed 's/.\(.*\)/\1/' | sed 's/\(.*\)./\1/') + if [[ $TRAVIS_JOB_NAME == 'media-similarity-tests' ]] + then + i=0 + NGROK_URL="" + ngrok config add-authtoken $NGROK_AUTH + ngrok config upgrade 2 + while [ -z "$NGROK_URL" -a $i -lt 5 ]; do + i=$(($i + 1)) + ngrok http 9000 >/dev/null & + until curl --silent -I -f --fail http://localhost:4040; do printf "."; sleep 10; done + curl -I -v http://localhost:4040 + curl localhost:4040/api/tunnels > ngrok.json + cat ngrok.json + NGROK_URL=$(grep -Po '"public_url": *\K"[^"]*"' ngrok.json | tail -n1 | sed 's/.\(.*\)/\1/' | sed 's/\(.*\)./\1/') + if [ -z $NGROK_URL ] + then + kill -9 $(pgrep ngrok) + fi + sleep 5 + done if [ -z $NGROK_URL ] then - kill -9 $(pgrep ngrok) + echo "Not able to connect a Ngrok Tunnel. Please try again!" + exit 1 fi - sleep 5 - done - if [ -z $NGROK_URL ] - then - echo "Not able to connect a Ngrok Tunnel. Please try again!" - exit 1 + echo "Ngrok tunnel: $NGROK_URL" + sed -i "s~similarity_media_file_url_host: ''~similarity_media_file_url_host: '$NGROK_URL'~g" check-api/config/config.yml + cat check-api/config/config.yml | grep similarity_media_file_url_host + docker-compose build web api api-background pender pender-background chromedriver alegre presto-server presto-audio presto-image presto-video + docker-compose -f docker-compose.yml -f docker-test.yml up -d web api api-background pender pender-background chromedriver alegre presto-server presto-audio presto-image presto-video + else + docker-compose build web api api-background pender pender-background chromedriver alegre presto-server presto-mean-tokens + docker-compose -f docker-compose.yml -f docker-test.yml up -d web api api-background pender pender-background chromedriver alegre presto-server presto-mean-tokens fi - echo "Ngrok tunnel: $NGROK_URL" - sed -i "s~similarity_media_file_url_host: ''~similarity_media_file_url_host: '$NGROK_URL'~g" check-api/config/config.yml - cat check-api/config/config.yml | grep similarity_media_file_url_host - docker-compose build - docker-compose -f docker-compose.yml -f docker-test.yml up -d until curl --silent -I -f --fail http://localhost:3100; do printf .; sleep 1; done until curl --silent -I -f --fail http://localhost:8000/ping; do printf .; sleep 1; done fi diff --git a/docker-compose.yml b/docker-compose.yml index f1b18b2a0e..af3e08a6ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -204,6 +204,21 @@ services: depends_on: elasticmq: condition: service_healthy + presto-mean-tokens: + build: presto + platform: linux/amd64 + volumes: + - "./presto:/app" + env_file: + - presto/.env_file + environment: + ROLE: worker + MODEL_NAME: mean_tokens.Model + networks: + - dev + depends_on: + elasticmq: + condition: service_healthy alegre: build: alegre ports: @@ -214,9 +229,6 @@ services: - redis - elasticsearch - presto-server - - presto-image - - presto-audio - - presto-video env_file: - alegre/.env_file networks: diff --git a/docker-test.yml b/docker-test.yml index 4d0e04be16..9df241aa11 100644 --- a/docker-test.yml +++ b/docker-test.yml @@ -54,6 +54,9 @@ services: presto-video: environment: BOILERPLATE_ENV: test + presto-mean-tokens: + environment: + BOILERPLATE_ENV: test web: environment: PLATFORM: web diff --git a/test/parallel-media-similarity-test.sh b/test/parallel-media-similarity-test.sh new file mode 100755 index 0000000000..b3d0110a8e --- /dev/null +++ b/test/parallel-media-similarity-test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +echo 'Starting tests...' + +FAIL=0 + +bundle exec rspec --tag bin8 spec/integration_spec.rb & + +for job in `jobs -p` +do + echo $job + wait $job || let "FAIL+=1" +done + +echo $FAIL + +if [ "$FAIL" == "0" ]; +then + exit 0 +else + exit 1 +fi \ No newline at end of file diff --git a/test/parallel-similarity-test.sh b/test/parallel-text-similarity-test.sh similarity index 100% rename from test/parallel-similarity-test.sh rename to test/parallel-text-similarity-test.sh diff --git a/test/run-tests.sh b/test/run-tests.sh index 40bbb54c37..7f64f54a9f 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -10,6 +10,10 @@ elif [[ $TRAVIS_JOB_NAME == 'unit-tests' ]] then npm run test:unit || exit 1 ./../scripts/uncovered-files -else - ./parallel-similarity-test.sh +elif [[ $TRAVIS_JOB_NAME == 'media-similarity-tests' ]] +then + ./parallel-media-similarity-test.sh +elif [[ $TRAVIS_JOB_NAME == 'text-similarity-tests' ]] +then + ./parallel-text-similarity-test.sh fi \ No newline at end of file diff --git a/test/spec/api_helpers.rb b/test/spec/api_helpers.rb index f60def77b6..401ad500c7 100644 --- a/test/spec/api_helpers.rb +++ b/test/spec/api_helpers.rb @@ -56,8 +56,10 @@ def api_create_team_and_bot(params = {}) user = params[:user] || api_register_and_login_with_email @slug = "test-team-#{Time.now.to_i}-#{rand(10_000).to_i}" team = request_api 'team', { name: "Test Team #{Time.now.to_i}", slug: @slug, email: user.email } - api_install_bot(params[:bot], team[:slug], params[:score]) if params[:bot] sleep 5 + puts "team created: #{team.inspect}" + api_install_bot(params[:bot], team[:slug], params[:score]) if params[:bot] + sleep 10 { user: user, team: team } end @@ -170,9 +172,12 @@ def api_suggest_similarity_between_items(team_id, source, target) end def api_install_bot(bot, slug = nil, settings = {}) + settings ||= { min_es_score: 0 } url = @driver.current_url.to_s team_slug = slug || url.match(%r{^https?://[^/]+/([^/]+)})[1] + puts "Installing bot with settings: #{settings.inspect}" request_api 'install_bot', { bot: bot, slug: team_slug, settings: settings.to_json } + sleep 2 @driver.navigate.to url end diff --git a/test/spec/app_spec_helpers.rb b/test/spec/app_spec_helpers.rb index 8985ad3539..8808aa8714 100644 --- a/test/spec/app_spec_helpers.rb +++ b/test/spec/app_spec_helpers.rb @@ -161,7 +161,7 @@ def create_media(url, wait_for_creation = true) def create_image(file) # show the side navigation for workspace tipline - wait_for_selector('#side-navigation__tipline-toggle').click + @driver.navigate.to "#{@config['self_url']}/#{@slug}/all-items" wait_for_selector('.projects-list') wait_for_selector('.projects-list__all-items').click wait_for_selector('#create-media-button__open-button').click diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 76e86dc9fa..998a846400 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -74,22 +74,29 @@ expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - it 'should identify videos as similar', bin7: true do - api_create_team_and_bot(bot: 'alegre') - @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" - create_image('files/video.mp4') - verbose_wait 5 - wait_for_selector('.cluster-card') - create_image('files/video2.mp4') - verbose_wait 5 - wait_for_selector('.cluster-card').click - wait_for_selector('.media__more-medias') - expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # it 'should identify texts as similar with vector search', bin7: true do + # data = api_create_team_and_bot(bot: 'alegre', score: { 'master_similarity_enabled' => true, 'text_similarity_enabled' => true, 'text_elasticsearch_matching_threshold' => 0.9, 'text_elasticsearch_suggestion_threshold' => 0.7, 'text_vector_matching_threshold' => 0.95, 'text_vector_suggestion_threshold' => 0.75, 'text_similarity_model' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'alegre_model_in_use' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'min_es_score' => 100_000 }) + # pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + # verbose_wait 3 + # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + # verbose_wait 3 + # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + # wait_for_selector('.media__more-medias') + # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + # end + # + + it 'should prepare environment for media similarity tests', bin8: true do + data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) + pm = api_create_claim(data: data, quote: 'Just kicking off Alegre service.') + sleep 60 # Wait for the item to be sent to Alegre + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + wait_for_selector('.quote-media-card') + expect(@driver.find_elements(:css, '.quote-media-card').size).to eq 1 end - it 'should identify images as similar', bin7: true do + it 'should identify images as similar', bin8: true do api_create_team_and_bot(bot: 'alegre') - @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" create_image('files/similarity.jpg') verbose_wait 4 wait_for_selector('.cluster-card') @@ -100,7 +107,7 @@ expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - it 'should extract text from a image', bin7: true do + it 'should extract text from a image', bin8: true do api_create_team_and_bot(bot: 'alegre') @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" create_image('files/ocr.png') @@ -111,7 +118,7 @@ expect(@driver.page_source.include?('Test')).to be(true) end - it 'should identify audios as similar', bin7: true do + it 'should identify audios as similar', bin8: true do api_create_team_and_bot(bot: 'alegre') @driver.navigate.to "#{@config['self_url']}/#{@slug}/settings/workspace" create_image('files/audio.mp3') @@ -123,4 +130,16 @@ wait_for_selector('.media__more-medias') expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end + + it 'should identify videos as similar', bin8: true do + api_create_team_and_bot(bot: 'alegre') + create_image('files/video.mp4') + verbose_wait 6 + wait_for_selector('.cluster-card') + create_image('files/video2.mp4') + verbose_wait 6 + wait_for_selector('.cluster-card').click + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + end end From baa19b177e03bb119cc99da40c32c96ad0b12e48 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:03:29 -0300 Subject: [PATCH 62/71] [full ci] From 719d6cdf58a7296f124becbc531e9f76cb54c7d8 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 15 Apr 2024 08:54:09 -0700 Subject: [PATCH 63/71] [full ci] CV2-4126 Run Video Integration Tests From 08625e4605127ba78d681e50ed352cc0fbac2da9 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Sat, 12 Oct 2024 08:34:02 -0700 Subject: [PATCH 64/71] fix --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index af3e08a6ca..72e3f4fa98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,6 +159,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-image: build: presto platform: linux/amd64 @@ -174,6 +175,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-audio: build: presto platform: linux/amd64 @@ -189,6 +191,7 @@ services: depends_on: elasticmq: condition: service_healthy + - redis presto-video: build: presto platform: linux/amd64 From 695098097e5966fd025388a9e7528495e2062d6c Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Sat, 12 Oct 2024 08:37:22 -0700 Subject: [PATCH 65/71] [full ci] readd vector tests --- test/spec/similarity_spec.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 998a846400..dd464bacc2 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -74,17 +74,17 @@ expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - # it 'should identify texts as similar with vector search', bin7: true do - # data = api_create_team_and_bot(bot: 'alegre', score: { 'master_similarity_enabled' => true, 'text_similarity_enabled' => true, 'text_elasticsearch_matching_threshold' => 0.9, 'text_elasticsearch_suggestion_threshold' => 0.7, 'text_vector_matching_threshold' => 0.95, 'text_vector_suggestion_threshold' => 0.75, 'text_similarity_model' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'alegre_model_in_use' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'min_es_score' => 100_000 }) - # pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') - # verbose_wait 3 - # api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') - # verbose_wait 3 - # @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" - # wait_for_selector('.media__more-medias') - # expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 - # end - # + it 'should identify texts as similar with vector search', bin7: true do + data = api_create_team_and_bot(bot: 'alegre', score: { 'master_similarity_enabled' => true, 'text_similarity_enabled' => true, 'text_elasticsearch_matching_threshold' => 0.9, 'text_elasticsearch_suggestion_threshold' => 0.7, 'text_vector_matching_threshold' => 0.95, 'text_vector_suggestion_threshold' => 0.75, 'text_similarity_model' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'alegre_model_in_use' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'min_es_score' => 100_000 }) + pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + verbose_wait 3 + api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + verbose_wait 3 + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + end + it 'should prepare environment for media similarity tests', bin8: true do data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) From f28c97693ece9c9d2cb531a8a85a88294bb2b851 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 14 Oct 2024 08:11:41 -0700 Subject: [PATCH 66/71] [full ci] run integration suite From 103f72bd4cef61432117187ee5b2e3d70149b074 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 14 Oct 2024 10:18:23 -0700 Subject: [PATCH 67/71] [full ci] remove redis which snuck in somehow --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 72e3f4fa98..f456b1b3f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-image: build: presto platform: linux/amd64 @@ -175,7 +174,6 @@ services: depends_on: elasticmq: condition: service_healthy - - redis presto-audio: build: presto platform: linux/amd64 From 8c73fba0ab1d3847fdab807915797c5f09639aeb Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 14 Oct 2024 11:53:14 -0700 Subject: [PATCH 68/71] [full ci] remove redis which snuck in somehow --- docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f456b1b3f6..86340bb78d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -187,8 +187,7 @@ services: networks: - dev depends_on: - elasticmq: - condition: service_healthy + - elasticmq - redis presto-video: build: presto From 1be52af07aaaaafe94b0ba5cc9188a2db44ae361 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 14 Oct 2024 14:56:17 -0700 Subject: [PATCH 69/71] remove whitespace --- test/spec/similarity_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index dd464bacc2..56ce50e8a8 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -85,7 +85,6 @@ expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 end - it 'should prepare environment for media similarity tests', bin8: true do data = api_create_team_and_bot(bot: 'alegre', score: { min_es_score: 0 }) pm = api_create_claim(data: data, quote: 'Just kicking off Alegre service.') From f616f04ddcc10cfdd9e7b2566ba72049d4b26472 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 14 Oct 2024 14:56:27 -0700 Subject: [PATCH 70/71] [full ci] run integration suite From 123b62828f0039f94ca2bc3c37a51d9c4f68934b Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 21 Oct 2024 11:48:14 -0700 Subject: [PATCH 71/71] add multiple vector model search --- test/spec/similarity_spec.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/spec/similarity_spec.rb b/test/spec/similarity_spec.rb index 56ce50e8a8..6a5cef1a6f 100644 --- a/test/spec/similarity_spec.rb +++ b/test/spec/similarity_spec.rb @@ -76,9 +76,20 @@ it 'should identify texts as similar with vector search', bin7: true do data = api_create_team_and_bot(bot: 'alegre', score: { 'master_similarity_enabled' => true, 'text_similarity_enabled' => true, 'text_elasticsearch_matching_threshold' => 0.9, 'text_elasticsearch_suggestion_threshold' => 0.7, 'text_vector_matching_threshold' => 0.95, 'text_vector_suggestion_threshold' => 0.75, 'text_similarity_model' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'alegre_model_in_use' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens'], 'min_es_score' => 100_000 }) - pm = api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry.') + pm = api_create_claim(data: data, quote: 'In January 2003, Batista joined Triple H, Ric Flair and Randy Orton to form the heel stable Evolution.[25] Batista, however, was sidelined for much of 2003 after he tore his right triceps muscle at a Raw live event in a tag team match alongside Orton against The Dudley Boyz.') verbose_wait 3 - api_create_claim(data: data, quote: 'Lorem Ipsum is used to generate dummy texts of the printing and IT industry!') + api_create_claim(data: data, quote: 'In January 2003, Batista joined Triple H, Ric Flair and Randy Orton to form the heel stable Evolution.[25] Batista, however, was sidelined for much of 2003 after he tore his right triceps muscle at a Raw live event in a tag team match alongside Orton against The Dudley Boy.') + verbose_wait 3 + @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" + wait_for_selector('.media__more-medias') + expect(@driver.find_elements(:css, '.media__relationship').size).to eq 1 + end + + it 'should identify texts as similar with vector search enabled on two models', bin7: true do + data = api_create_team_and_bot(bot: 'alegre', score: { 'master_similarity_enabled' => true, 'text_similarity_enabled' => true, 'text_elasticsearch_matching_threshold' => 0.9, 'text_elasticsearch_suggestion_threshold' => 0.7, 'text_vector_matching_threshold' => 0.95, 'text_vector_suggestion_threshold' => 0.75, 'text_similarity_model' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens', 'paraphrase-multilingual-mpnet-base-v2'], 'alegre_model_in_use' => ['elasticsearch', 'xlm-r-bert-base-nli-stsb-mean-tokens', 'paraphrase-multilingual-mpnet-base-v2'], 'min_es_score' => 100_000 }) + pm = api_create_claim(data: data, quote: 'The ends of the warp threads are usually fastened to beams. One end is fastened to one beam, the other end to a second beam, so that the warp threads all lie parallel and are all the same length. The beams are held apart to keep the warp threads taut.') + verbose_wait 3 + api_create_claim(data: data, quote: 'The ends of the warp threads are usually fastened to beams. One end is fastened to one beam, the other end to a second beam, so that the warp threads all lie parallel and are all the same length. The beams are held apart to keep the warp threads tight.') verbose_wait 3 @driver.navigate.to "#{@config['self_url']}/#{data[:team].slug}/media/#{pm.id}" wait_for_selector('.media__more-medias')