From d40c55a491c1067a5f2e49ffeb28a93eb89a993c Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Sat, 23 Mar 2024 07:42:22 +1100 Subject: [PATCH 01/27] ci: fix push to use published dev container for unit tests --- .github/workflows/push.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6e67ea80c..5484e145b 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -19,6 +19,7 @@ env: DF_ENCRYPTION_DETERMINISTIC_KEY: "anlmuJ6cB3bN3biXRbYvmPsC5ALPFqGG" DF_ENCRYPTION_KEY_DERIVATION_SALT: "hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10" + jobs: unit-tests: runs-on: ubuntu-latest @@ -34,17 +35,17 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - - name: Build base doubtfire-api development image - uses: docker/build-push-action@v3 - with: - context: . - push: false - load: true - tags: doubtfire-api-development:local + # - name: Build base doubtfire-api development image + # uses: docker/build-push-action@v3 + # with: + # context: . + # push: false + # load: true + # tags: doubtfire-api-development:local - name: Populate database uses: addnab/docker-run-action@v3 with: - image: doubtfire-api-development:local + image: lmsdoubtfire/formatif-devcontainer:8.3-dev options: > -v ${{ github.workspace }}:/doubtfire -e RAILS_ENV @@ -67,7 +68,7 @@ jobs: - name: Run rake tests uses: addnab/docker-run-action@v3 with: - image: doubtfire-api-development:local + image: lmsdoubtfire/formatif-devcontainer:8.3-dev options: > -v ${{ github.workspace }}:/doubtfire -e RAILS_ENV From 2beb6e8599cf6b99992e34548615e7802e7ff141 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Wed, 1 May 2024 05:50:47 +1000 Subject: [PATCH 02/27] fix: correct tii migrationm defaults --- db/migrate/20240105055902_add_tii_details.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrate/20240105055902_add_tii_details.rb b/db/migrate/20240105055902_add_tii_details.rb index 5f7f0377d..95cef1530 100644 --- a/db/migrate/20240105055902_add_tii_details.rb +++ b/db/migrate/20240105055902_add_tii_details.rb @@ -70,8 +70,8 @@ def change t.integer :error_code t.text :custom_error_message - t.text :log, default: [] - t.text :params, default: {} + t.text :log, default: "[]" + t.text :params, default: "{}" t.timestamps From 1fa7b0b10f855bc2e23aa27e78ed41ff3e5b4683 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Wed, 1 May 2024 08:56:03 +1000 Subject: [PATCH 03/27] fix: add missing moss language in task def post --- app/api/task_definitions_api.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/api/task_definitions_api.rb b/app/api/task_definitions_api.rb index de1039b55..c5a065cd4 100644 --- a/app/api/task_definitions_api.rb +++ b/app/api/task_definitions_api.rb @@ -32,6 +32,7 @@ class TaskDefinitionsApi < Grape::API requires :max_quality_pts, type: Integer, desc: 'A range for quality points when quality is assessed' optional :assessment_enabled, type: Boolean, desc: 'Enable or disable assessment' optional :overseer_image_id, type: Integer, desc: 'The id of the Docker image for overseer' + optional :moss_language, type: String, desc: 'The language to use for code similarity checks' end end post '/units/:unit_id/task_definitions/' do @@ -59,7 +60,8 @@ class TaskDefinitionsApi < Grape::API :is_graded, :max_quality_pts, :assessment_enabled, - :overseer_image_id + :overseer_image_id, + :moss_language ) task_params[:unit_id] = unit.id @@ -110,6 +112,7 @@ class TaskDefinitionsApi < Grape::API optional :max_quality_pts, type: Integer, desc: 'A range for quality points when quality is assessed' optional :assessment_enabled, type: Boolean, desc: 'Enable or disable assessment' optional :overseer_image_id, type: Integer, desc: 'The id of the Docker image name for overseer' + optional :moss_language, type: String, desc: 'The language to use for code similarity checks' end end put '/units/:unit_id/task_definitions/:id' do @@ -136,7 +139,8 @@ class TaskDefinitionsApi < Grape::API :is_graded, :max_quality_pts, :assessment_enabled, - :overseer_image_id + :overseer_image_id, + :moss_language ) task_params[:upload_requirements] = JSON.parse(params[:task_def][:upload_requirements]) unless params[:task_def][:plagiarism_checks].nil? @@ -178,6 +182,7 @@ class TaskDefinitionsApi < Grape::API end end + puts task_def.upload_requirements present task_def, with: Entities::TaskDefinitionEntity, my_role: unit.role_for(current_user) end From 19107bf87601115ea15036f25634b2ba30e23c7c Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 08:59:46 +1000 Subject: [PATCH 04/27] fix: remove plagiarism checks field --- app/api/entities/task_definition_entity.rb | 1 - app/api/task_definitions_api.rb | 8 +-- app/models/task_definition.rb | 55 +------------------- db/migrate/20240105055902_add_tii_details.rb | 2 + test/api/units/task_definitions_api_test.rb | 2 - test/models/task_definition_test.rb | 2 +- 6 files changed, 7 insertions(+), 63 deletions(-) diff --git a/app/api/entities/task_definition_entity.rb b/app/api/entities/task_definition_entity.rb index 780dc5c83..94ba180d4 100644 --- a/app/api/entities/task_definition_entity.rb +++ b/app/api/entities/task_definition_entity.rb @@ -33,7 +33,6 @@ def staff?(my_role) expose :tutorial_stream_abbr do |task_definition, options| task_definition.tutorial_stream.abbreviation unless task_definition.tutorial_stream.nil? end - expose :plagiarism_checks, if: ->(unit, options) { staff?(options[:my_role]) } expose :plagiarism_warn_pct, if: ->(unit, options) { staff?(options[:my_role]) } expose :restrict_status_updates, if: ->(unit, options) { staff?(options[:my_role]) } expose :group_set_id diff --git a/app/api/task_definitions_api.rb b/app/api/task_definitions_api.rb index c5a065cd4..4f0853821 100644 --- a/app/api/task_definitions_api.rb +++ b/app/api/task_definitions_api.rb @@ -26,7 +26,6 @@ class TaskDefinitionsApi < Grape::API requires :abbreviation, type: String, desc: 'The abbreviation of the task' requires :restrict_status_updates, type: Boolean, desc: 'Restrict updating of the status to staff' optional :upload_requirements, type: String, desc: 'Task file upload requirements' - optional :plagiarism_checks, type: String, desc: 'The list of checks to perform' requires :plagiarism_warn_pct, type: Integer, desc: 'The percent at which to record and warn about plagiarism' requires :is_graded, type: Boolean, desc: 'Whether or not this task definition is a graded task' requires :max_quality_pts, type: Integer, desc: 'A range for quality points when quality is assessed' @@ -65,8 +64,7 @@ class TaskDefinitionsApi < Grape::API ) task_params[:unit_id] = unit.id - task_params[:upload_requirements] = JSON.parse(params[:task_def][:upload_requirements]) unless params[:task_def][:plagiarism_checks].nil? - task_params[:plagiarism_checks] = JSON.parse(params[:task_def][:plagiarism_checks]) unless params[:task_def][:plagiarism_checks].nil? + task_params[:upload_requirements] = JSON.parse(params[:task_def][:upload_requirements]) unless params[:task_def][:upload_requirements].nil? task_def = TaskDefinition.new(task_params) @@ -106,7 +104,6 @@ class TaskDefinitionsApi < Grape::API optional :abbreviation, type: String, desc: 'The abbreviation of the task' optional :restrict_status_updates, type: Boolean, desc: 'Restrict updating of the status to staff' optional :upload_requirements, type: String, desc: 'Task file upload requirements' - optional :plagiarism_checks, type: String, desc: 'The list of checks to perform' optional :plagiarism_warn_pct, type: Integer, desc: 'The percent at which to record and warn about plagiarism' optional :is_graded, type: Boolean, desc: 'Whether or not this task definition is a graded task' optional :max_quality_pts, type: Integer, desc: 'A range for quality points when quality is assessed' @@ -143,8 +140,7 @@ class TaskDefinitionsApi < Grape::API :moss_language ) - task_params[:upload_requirements] = JSON.parse(params[:task_def][:upload_requirements]) unless params[:task_def][:plagiarism_checks].nil? - task_params[:plagiarism_checks] = JSON.parse(params[:task_def][:plagiarism_checks]) unless params[:task_def][:plagiarism_checks].nil? + task_params[:upload_requirements] = JSON.parse(params[:task_def][:upload_requirements]) unless params[:task_def][:upload_requirements].nil? # Ensure changes to a TD defined as a "draft task definition" are validated if unit.draft_task_definition_id == params[:id] diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index 04d44439c..734673d03 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -38,7 +38,6 @@ class TaskDefinition < ApplicationRecord validates :max_quality_pts, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 100, message: 'must be between 0 and 100' } validate :upload_requirements, :check_upload_requirements_format - validate :plagiarism_checks, :check_plagiarism_format validates :description, length: { maximum: 4095, allow_blank: true } @@ -148,54 +147,6 @@ def docker_image_name_tag overseer_image.tag end - def check_plagiarism_format() - return if plagiarism_checks.nil? - - json_data = self.plagiarism_checks - - # ensure we have a structure that is : [ { "key": "...", "type": "...", "pattern": "..."}, { ... } ] - unless json_data.class == Array - errors.add(:plagiarism_checks, 'is not in a valid format! Should be [ { "key": "...", "type": "...", "pattern": "..."}, { ... } ]. Did not contain array.') - return - end - - # Loop through checks in the array - i = 0 - for req in json_data do - # They must be json objects - unless req.class == Hash - errors.add(:plagiarism_checks, "is not in a valid format! Should be [ { \"key\": \"...\", \"type\": \"...\", \"pattern\": \"...\"}, { ... } ]. Array did not contain hashes for item #{i + 1}..") - return - end - - # They must have these keys... - if (!req.key? 'key') || (!req.key? 'type') || (!req.key? 'pattern') - errors.add(:plagiarism_checks, "is not in a valid format! Should be [ { \"key\": \"...\", \"type\": \"...\", \"pattern\": \"...\"}, { ... } ]. Missing a key for item #{i + 1}.") - return - end - - # Validate the type (MOSS now, Turnitin later) - if (!req['type'].match(/^moss /)) - errors.add(:plagiarism_checks, "does not have a valid type.") - return - end - - # Check patter to exclude any path separators - if (req['pattern'].match(/(\/)|([.][.])/)) - errors.add(:plagiarism_checks, " pattern contains invalid characters.") - return - end - - # Check only key, type, and pattern keys are present - if req.keys.length > 3 - errors.add(:plagiarism_checks, "has additional values for item #{i + 1} --> #{req.keys.join(' ')}.") - end - - # Move to the next check - i += 1 - end - end - def glob_for_upload_requirement(idx) "#{idx.to_s.rjust(3, '0')}-#{upload_requirements[idx]['type']}.*" end @@ -329,10 +280,9 @@ def propogate_date_changes date_diff def to_csv_row TaskDefinition.csv_columns - .reject { |col| [:start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :upload_requirements, :plagiarism_checks, :group_set, :tutorial_stream].include? col } + .reject { |col| [:start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :upload_requirements, :group_set, :tutorial_stream].include? col } .map { |column| attributes[column.to_s] } + [ - plagiarism_checks.to_json, group_set.nil? ? "" : group_set.name, upload_requirements.to_s, start_week, @@ -348,7 +298,7 @@ def to_csv_row end def self.csv_columns - [:name, :abbreviation, :description, :weighting, :target_grade, :restrict_status_updates, :max_quality_pts, :is_graded, :plagiarism_warn_pct, :plagiarism_checks, :group_set, :upload_requirements, :start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :tutorial_stream] + [:name, :abbreviation, :description, :weighting, :target_grade, :restrict_status_updates, :max_quality_pts, :is_graded, :plagiarism_warn_pct, :group_set, :upload_requirements, :start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :tutorial_stream] end def self.task_def_for_csv_row(unit, row) @@ -395,7 +345,6 @@ def self.task_def_for_csv_row(unit, row) result.due_date = due_date result.plagiarism_warn_pct = row[:plagiarism_warn_pct].to_i - result.plagiarism_checks = JSON.parse(row[:plagiarism_checks]) unless row[:plagiarism_checks].nil? if row[:group_set].present? result.group_set = unit.group_sets.where(name: row[:group_set]).first diff --git a/db/migrate/20240105055902_add_tii_details.rb b/db/migrate/20240105055902_add_tii_details.rb index 95cef1530..f3e0526c2 100644 --- a/db/migrate/20240105055902_add_tii_details.rb +++ b/db/migrate/20240105055902_add_tii_details.rb @@ -91,5 +91,7 @@ def change task_definition.save end end + + remove_column :task_definitions, :plagiarism_checks, :string end end diff --git a/test/api/units/task_definitions_api_test.rb b/test/api/units/task_definitions_api_test.rb index 9eb08fe2f..656d2d67c 100644 --- a/test/api/units/task_definitions_api_test.rb +++ b/test/api/units/task_definitions_api_test.rb @@ -47,7 +47,6 @@ def test_task_definition_cud abbreviation: 'P1.1', restrict_status_updates: false, upload_requirements: '[ { "key": "file0", "name": "Shape Class", "type": "document" } ]', - plagiarism_checks: '[]', plagiarism_warn_pct: 80, is_graded: false, max_quality_pts: 0 @@ -82,7 +81,6 @@ def test_task_definition_cud abbreviation: 'P1.2', restrict_status_updates: true, upload_requirements: [ { "key": "file0", "name": "Other Class", "type": "document" } ].to_json, - plagiarism_checks: [].to_json, plagiarism_warn_pct: 80, is_graded: false, max_quality_pts: 0 diff --git a/test/models/task_definition_test.rb b/test/models/task_definition_test.rb index 12a2cea2e..d20634d8c 100644 --- a/test/models/task_definition_test.rb +++ b/test/models/task_definition_test.rb @@ -145,7 +145,7 @@ def test_export_task_definitions_csv task_defs_csv = CSV.parse unit.task_definitions_csv, headers: true task_defs_csv.each do |task_def_csv| task_def = unit.task_definitions.find_by(abbreviation: task_def_csv['abbreviation']) - keys_to_ignore = ['tutorial_stream', 'plagiarism_checks', 'start_week', 'start_day', 'target_week', 'target_day', 'due_week', 'due_day'] + keys_to_ignore = ['tutorial_stream', 'start_week', 'start_day', 'target_week', 'target_day', 'due_week', 'due_day'] task_def_csv.each do |key, value| unless keys_to_ignore.include?(key) assert_equal(task_def[key].to_s, value) From 1962cc96ff46134d756984473d1022792e9ada1a Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 09:14:53 +1000 Subject: [PATCH 05/27] fix: remove serialisation of plagiarism checks --- app/models/task_definition.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index 734673d03..7e78bd2a7 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -1,11 +1,6 @@ require 'json' class TaskDefinition < ApplicationRecord - # Record triggers - before associations - after_update do |_td| - clear_related_plagiarism if plagiarism_checks.nil? && moss_similarities? - end - before_destroy :delete_associated_files after_update :move_files_on_abbreviation_change, if: :saved_change_to_abbreviation? @@ -28,7 +23,6 @@ class TaskDefinition < ApplicationRecord has_many :tii_actions, as: :entity, dependent: :destroy serialize :upload_requirements, coder: JSON - serialize :plagiarism_checks, coder: JSON # Model validations/constraints validates :name, uniqueness: { scope: :unit_id } # task definition names within a unit must be unique From a2b7343c8f488555563f02916dd1e6b9e47dfff5 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 09:16:02 +1000 Subject: [PATCH 06/27] chore(release): 8.0.0-4 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7da7dd36..16c5bd373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-4](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-3...v8.0.0-4) (2024-05-01) + + +### Bug Fixes + +* add missing moss language in task def post ([1fa7b0b](https://github.com/macite/doubtfire-deploy/commit/1fa7b0b10f855bc2e23aa27e78ed41ff3e5b4683)) +* correct tii migrationm defaults ([2beb6e8](https://github.com/macite/doubtfire-deploy/commit/2beb6e8599cf6b99992e34548615e7802e7ff141)) +* remove plagiarism checks field ([19107bf](https://github.com/macite/doubtfire-deploy/commit/19107bf87601115ea15036f25634b2ba30e23c7c)) +* remove serialisation of plagiarism checks ([1962cc9](https://github.com/macite/doubtfire-deploy/commit/1962cc96ff46134d756984473d1022792e9ada1a)) + ## [8.0.0-3](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-2...v8.0.0-3) (2024-03-22) From d1ab30ba666898f556b69db53766124742b4f593 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 15:33:19 +1000 Subject: [PATCH 07/27] fix: correct updates in TII migration --- db/migrate/20240105055902_add_tii_details.rb | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/db/migrate/20240105055902_add_tii_details.rb b/db/migrate/20240105055902_add_tii_details.rb index f3e0526c2..42a0bee33 100644 --- a/db/migrate/20240105055902_add_tii_details.rb +++ b/db/migrate/20240105055902_add_tii_details.rb @@ -17,9 +17,6 @@ def change remove_column :tasks, :max_pct_similar, :integer remove_column :projects, :max_pct_similar, :integer - TaskSimilarity.update_all(type: 'MossTaskSimilarity') - # TaskSimilarity.update_all('flagged = not flagged') - create_table :tii_submissions do |t| t.references :task, null: false t.references :tii_task_similarity, null: true @@ -79,11 +76,19 @@ def change t.index :complete end + TaskSimilarity.update_all(type: 'MossTaskSimilarity') + # TaskSimilarity.update_all('flagged = not flagged') + TaskDefinition.find_in_batches do |batch| + require 'json' + batch.each do |task_definition| - next unless task_definition.plagiarism_checks.present? && task_definition.plagiarism_checks.any? + next unless task_definition.plagiarism_checks.present? + plagiarism_checks = JSON.parse(task_definition.plagiarism_checks) + + next unless plagiarism_checks.any? - task_definition.update(moss_language: task_definition.plagiarism_checks.first['type']) + task_definition.update(moss_language: plagiarism_checks.first['type']) task_definition.upload_requirements.each do |upload_requirement| next unless upload_requirement['type'] == 'code' upload_requirement['tii_check'] = true From 354b5b2668b256af7312bc281adc46e9e1337c5c Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 15:37:13 +1000 Subject: [PATCH 08/27] chore: update schema with changed tii details --- db/schema.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 0f10c135b..645806173 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -237,7 +237,6 @@ t.string "upload_requirements", limit: 4096 t.integer "target_grade", default: 0 t.boolean "restrict_status_updates", default: false - t.string "plagiarism_checks", limit: 4096 t.string "plagiarism_report_url" t.boolean "plagiarism_updated", default: false t.integer "plagiarism_warn_pct", default: 50 @@ -356,15 +355,13 @@ t.boolean "retry", default: true, null: false t.integer "error_code" t.text "custom_error_message" - t.text "log", size: :long, default: "[]", collation: "utf8mb4_bin" - t.text "params", size: :long, default: "{}", collation: "utf8mb4_bin" + t.text "log", default: "[]" + t.text "params", default: "{}" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["complete"], name: "index_tii_actions_on_complete" t.index ["entity_type", "entity_id"], name: "index_tii_actions_on_entity" t.index ["retry"], name: "index_tii_actions_on_retry" - t.check_constraint "json_valid(`log`)", name: "log" - t.check_constraint "json_valid(`params`)", name: "params" end create_table "tii_group_attachments", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| From a2e92aa4986dcf4712561108800f00ba510c45e1 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 19:54:27 +1000 Subject: [PATCH 09/27] chore(release): 8.0.0-5 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c5bd373..1c4174774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-5](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-4...v8.0.0-5) (2024-05-02) + + +### Bug Fixes + +* correct updates in TII migration ([d1ab30b](https://github.com/macite/doubtfire-deploy/commit/d1ab30ba666898f556b69db53766124742b4f593)) + ## [8.0.0-4](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-3...v8.0.0-4) (2024-05-01) From 73fcbe3f5adb603253033e7b126502a5d3c006f1 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 20:18:20 +1000 Subject: [PATCH 10/27] fix: revert to doubtfire local image for unit tests --- .github/workflows/push.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index cba75f146..482af4d78 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -28,7 +28,6 @@ env: DF_ENCRYPTION_KEY_DERIVATION_SALT: "hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10" DF_REDIS_SIDEKIQ_URL: "redis://redis:6379/0" - jobs: unit-tests: runs-on: ubuntu-latest @@ -62,7 +61,7 @@ jobs: - name: Populate database uses: addnab/docker-run-action@v3 with: - image: lmsdoubtfire/formatif-devcontainer:8.3-dev + image: doubtfire-api-development:local options: > -v ${{ github.workspace }}:/doubtfire -e RAILS_ENV @@ -86,7 +85,7 @@ jobs: - name: Run unit tests uses: addnab/docker-run-action@v3 with: - image: lmsdoubtfire/formatif-devcontainer:8.3-dev + image: doubtfire-api-development:local options: > -v ${{ github.workspace }}:/doubtfire -e RAILS_ENV From b70e5b5e4ddcffc20b26669ceb1b08e63da97289 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 20:26:42 +1000 Subject: [PATCH 11/27] chore: update bundle to latest gems --- Gemfile.lock | 277 ++++++++++++++++++++++++++------------------------- 1 file changed, 142 insertions(+), 135 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 99dadcc6f..3bd5a04f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,35 +2,35 @@ GEM remote: https://rubygems.org/ specs: Ascii85 (1.1.0) - actioncable (7.1.2) - actionpack (= 7.1.2) - activesupport (= 7.1.2) + actioncable (7.1.3.2) + actionpack (= 7.1.3.2) + activesupport (= 7.1.3.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.2) - actionpack (= 7.1.2) - activejob (= 7.1.2) - activerecord (= 7.1.2) - activestorage (= 7.1.2) - activesupport (= 7.1.2) + actionmailbox (7.1.3.2) + actionpack (= 7.1.3.2) + activejob (= 7.1.3.2) + activerecord (= 7.1.3.2) + activestorage (= 7.1.3.2) + activesupport (= 7.1.3.2) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.1.2) - actionpack (= 7.1.2) - actionview (= 7.1.2) - activejob (= 7.1.2) - activesupport (= 7.1.2) + actionmailer (7.1.3.2) + actionpack (= 7.1.3.2) + actionview (= 7.1.3.2) + activejob (= 7.1.3.2) + activesupport (= 7.1.3.2) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.2) - actionpack (7.1.2) - actionview (= 7.1.2) - activesupport (= 7.1.2) + actionpack (7.1.3.2) + actionview (= 7.1.3.2) + activesupport (= 7.1.3.2) nokogiri (>= 1.8.5) racc rack (>= 2.2.4) @@ -38,35 +38,35 @@ GEM rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.2) - actionpack (= 7.1.2) - activerecord (= 7.1.2) - activestorage (= 7.1.2) - activesupport (= 7.1.2) + actiontext (7.1.3.2) + actionpack (= 7.1.3.2) + activerecord (= 7.1.3.2) + activestorage (= 7.1.3.2) + activesupport (= 7.1.3.2) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.2) - activesupport (= 7.1.2) + actionview (7.1.3.2) + activesupport (= 7.1.3.2) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.2) - activesupport (= 7.1.2) + activejob (7.1.3.2) + activesupport (= 7.1.3.2) globalid (>= 0.3.6) - activemodel (7.1.2) - activesupport (= 7.1.2) - activerecord (7.1.2) - activemodel (= 7.1.2) - activesupport (= 7.1.2) + activemodel (7.1.3.2) + activesupport (= 7.1.3.2) + activerecord (7.1.3.2) + activemodel (= 7.1.3.2) + activesupport (= 7.1.3.2) timeout (>= 0.4.0) - activestorage (7.1.2) - actionpack (= 7.1.2) - activejob (= 7.1.2) - activerecord (= 7.1.2) - activesupport (= 7.1.2) + activestorage (7.1.3.2) + actionpack (= 7.1.3.2) + activejob (= 7.1.3.2) + activerecord (= 7.1.3.2) + activesupport (= 7.1.3.2) marcel (~> 1.0) - activesupport (7.1.2) + activesupport (7.1.3.2) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) @@ -90,9 +90,9 @@ GEM erubi (>= 1.0.0) rack (>= 0.9.0) rouge (>= 1.0.0) - bigdecimal (3.1.5) - bindata (2.4.15) - bootsnap (1.17.0) + bigdecimal (3.1.7) + bindata (2.5.0) + bootsnap (1.18.3) msgpack (~> 1.2) builder (3.2.4) bunny (2.22.0) @@ -107,9 +107,10 @@ GEM code_analyzer (0.5.5) sexp_processor coderay (1.1.3) - concurrent-ruby (1.2.2) + concurrent-ruby (1.2.3) connection_pool (2.4.1) - crack (0.4.5) + crack (1.0.0) + bigdecimal rexml crass (1.0.6) database_cleaner-active_record (2.1.0) @@ -117,7 +118,7 @@ GEM database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) date (3.3.4) - devise (4.9.3) + devise (4.9.4) bcrypt (~> 3.0) orm_adapter (~> 0.1) railties (>= 4.1.0) @@ -126,15 +127,14 @@ GEM devise_ldap_authenticatable (0.8.7) devise (>= 3.4.1) net-ldap (>= 0.16.0) - diff-lcs (1.5.0) + diff-lcs (1.5.1) docile (1.4.0) - domain_name (0.6.20231109) - dotenv (2.8.1) - dotenv-rails (2.8.1) - dotenv (= 2.8.1) - railties (>= 3.2) - drb (2.2.0) - ruby2_keywords + domain_name (0.6.20240107) + dotenv (3.1.1) + dotenv-rails (3.1.1) + dotenv (= 3.1.1) + railties (>= 6.1) + drb (2.2.1) dry-core (1.0.1) concurrent-ruby (~> 1.0) zeitwerk (~> 2.6) @@ -143,7 +143,8 @@ GEM concurrent-ruby (~> 1.0) dry-core (~> 1.0, < 2) zeitwerk (~> 2.6) - dry-types (1.7.1) + dry-types (1.7.2) + bigdecimal (~> 3.0) concurrent-ruby (~> 1.0) dry-core (~> 1.0) dry-inflector (~> 1.0) @@ -152,27 +153,26 @@ GEM e2mmap (0.1.0) erubi (1.12.0) erubis (2.7.0) - et-orbi (1.2.7) + et-orbi (1.2.11) tzinfo ethon (0.16.0) ffi (>= 1.15.0) - factory_bot (6.4.4) + factory_bot (6.4.6) activesupport (>= 5.0.0) - factory_bot_rails (6.4.2) + factory_bot_rails (6.4.3) factory_bot (~> 6.4) railties (>= 5.0.0) - faker (3.2.2) + faker (3.3.1) i18n (>= 1.8.11, < 2) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) + faraday (2.9.0) + faraday-net_http (>= 2.0, < 3.2) faraday-follow_redirects (0.3.0) faraday (>= 1, < 3) - faraday-net_http (3.0.2) + faraday-net_http (3.1.0) + net-http ffi (1.16.3) - fugit (1.9.0) - et-orbi (~> 1, >= 1.2.7) + fugit (1.11.0) + et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) @@ -183,13 +183,13 @@ GEM mustermann-grape (~> 1.0.0) rack (>= 1.3.0) rack-accept - grape-entity (1.0.0) + grape-entity (1.0.1) activesupport (>= 3.0.0) multi_json (>= 1.3.2) - grape-swagger (2.0.0) + grape-swagger (2.0.3) grape (>= 1.7, < 3.0) rack-test (~> 2) - grape-swagger-rails (0.4.0) + grape-swagger-rails (0.5.0) railties (>= 6.0.6.1) hashdiff (1.1.0) hashery (2.1.2) @@ -197,20 +197,21 @@ GEM http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) - i18n (1.14.1) + i18n (1.14.4) concurrent-ruby (~> 1.0) icalendar (2.10.1) ice_cube (~> 0.16) ice_cube (0.16.4) - io-console (0.7.1) - irb (1.11.0) - rdoc - reline (>= 0.3.8) + io-console (0.7.2) + irb (1.13.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) jaro_winkler (1.5.6) - json (2.7.1) - json-jwt (1.16.4) + json (2.7.2) + json-jwt (1.16.6) activesupport (>= 4.2) aes_key_wrap + base64 bindata faraday (~> 2.0) faraday-follow_redirects @@ -219,7 +220,7 @@ GEM kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) language_server-protocol (3.17.0.3) - listen (3.8.0) + listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) loofah (2.22.0) @@ -230,12 +231,12 @@ GEM net-imap net-pop net-smtp - marcel (1.0.2) - mime-types (3.5.1) + marcel (1.0.4) + mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2023.1205) + mime-types-data (3.2024.0305) mini_mime (1.1.5) - minitest (5.20.0) + minitest (5.22.3) minitest-around (0.5.0) minitest (~> 5.0) minitest-rails (7.1.0) @@ -250,26 +251,29 @@ GEM mustermann-grape (1.0.2) mustermann (>= 1.0.0) mutex_m (0.2.0) - mysql2 (0.5.5) - net-imap (0.4.9) + mysql2 (0.5.6) + net-http (0.4.1) + uri + net-imap (0.4.10) date net-protocol - net-ldap (0.18.0) + net-ldap (0.19.0) net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-smtp (0.4.0) + net-smtp (0.5.0) net-protocol netrc (0.11.0) - nio4r (2.7.0) - nokogiri (1.15.5-aarch64-linux) + nio4r (2.7.1) + nokogiri (1.16.4-aarch64-linux) racc (~> 1.4) - nokogiri (1.15.5-x86_64-linux) + nokogiri (1.16.4-x86_64-linux) racc (~> 1.4) + observer (0.1.2) orm_adapter (0.5.0) parallel (1.24.0) - parser (3.2.2.4) + parser (3.3.1.0) ast (~> 2.4.1) racc pdf-reader (2.12.0) @@ -279,18 +283,18 @@ GEM ruby-rc4 ttfunk pkg-config (1.5.6) - prism (0.24.0) + prism (0.27.0) psych (5.1.2) stringio - public_suffix (5.0.4) - puma (6.4.0) + public_suffix (5.0.5) + puma (6.4.2) nio4r (~> 2.0) raabro (1.4.0) racc (1.7.3) - rack (3.0.8) + rack (3.0.10) rack-accept (0.4.5) rack (>= 0.4) - rack-cors (2.0.1) + rack-cors (2.0.2) rack (>= 2.0.0) rack-session (2.0.0) rack (>= 3.0.0) @@ -299,20 +303,20 @@ GEM rackup (2.1.0) rack (>= 3) webrick (~> 1.8) - rails (7.1.2) - actioncable (= 7.1.2) - actionmailbox (= 7.1.2) - actionmailer (= 7.1.2) - actionpack (= 7.1.2) - actiontext (= 7.1.2) - actionview (= 7.1.2) - activejob (= 7.1.2) - activemodel (= 7.1.2) - activerecord (= 7.1.2) - activestorage (= 7.1.2) - activesupport (= 7.1.2) + rails (7.1.3.2) + actioncable (= 7.1.3.2) + actionmailbox (= 7.1.3.2) + actionmailer (= 7.1.3.2) + actionpack (= 7.1.3.2) + actiontext (= 7.1.3.2) + actionview (= 7.1.3.2) + activejob (= 7.1.3.2) + activemodel (= 7.1.3.2) + activerecord (= 7.1.3.2) + activestorage (= 7.1.3.2) + activesupport (= 7.1.3.2) bundler (>= 1.15.0) - railties (= 7.1.2) + railties (= 7.1.3.2) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -330,29 +334,29 @@ GEM json require_all (~> 3.0) ruby-progressbar - railties (7.1.2) - actionpack (= 7.1.2) - activesupport (= 7.1.2) + railties (7.1.3.2) + actionpack (= 7.1.3.2) + activesupport (= 7.1.3.2) irb rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) zeitwerk (~> 2.6) rainbow (3.1.1) - rake (13.1.0) + rake (13.2.1) rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) rbs (2.8.4) rbtree (0.4.6) - rdoc (6.6.2) + rdoc (6.6.3.1) psych (>= 4.0.0) - redis (5.0.8) - redis-client (>= 0.17.0) - redis-client (0.19.1) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) connection_pool - regexp_parser (2.8.3) - reline (0.4.1) + regexp_parser (2.9.0) + reline (0.5.4) io-console (~> 0.5) require_all (3.0.0) responders (3.1.1) @@ -366,7 +370,8 @@ GEM reverse_markdown (2.1.1) nokogiri rexml (3.2.6) - rmagick (5.3.0) + rmagick (5.5.0) + observer (~> 0.1) pkg-config (~> 1.4) roo (2.7.1) nokogiri (~> 1) @@ -375,34 +380,34 @@ GEM nokogiri roo (>= 2.0.0, < 3) spreadsheet (> 0.9.0) - rouge (4.2.0) - rubocop (1.59.0) + rouge (4.2.1) + rubocop (1.63.4) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.2.2.4) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) rubocop-faker (1.1.0) faker (>= 2.12.0) rubocop (>= 0.82.0) - rubocop-rails (2.23.1) + rubocop-rails (2.24.1) activesupport (>= 4.2.0) rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) - rubocop-ast (>= 1.30.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-filemagic (0.7.3) - ruby-lsp (0.14.6) + ruby-lsp (0.16.6) language_server-protocol (~> 3.17.0) - prism (>= 0.22.0, < 0.25) + prism (>= 0.23.0, < 0.28) sorbet-runtime (>= 0.5.10782) - ruby-ole (1.2.12.2) + ruby-ole (1.2.13.1) ruby-progressbar (1.13.0) ruby-rc4 (0.1.5) ruby-saml (1.13.0) @@ -411,12 +416,12 @@ GEM ruby2_keywords (0.0.5) rubyzip (1.3.0) set (1.1.0) - sexp_processor (4.17.0) - sidekiq (7.2.0) + sexp_processor (4.17.1) + sidekiq (7.2.4) concurrent-ruby (< 2) connection_pool (>= 2.3.0) rack (>= 2.2.4) - redis-client (>= 0.14.0) + redis-client (>= 0.19.0) sidekiq-cron (1.12.0) fugit (~> 1.8) globalid (>= 1.0.1) @@ -443,11 +448,12 @@ GEM thor (~> 1.0) tilt (~> 2.0) yard (~> 0.9, >= 0.9.24) - sorbet-runtime (0.5.11307) + sorbet-runtime (0.5.11367) sorted_set (1.0.3) rbtree set (~> 1.0) - spreadsheet (1.3.0) + spreadsheet (1.3.1) + bigdecimal ruby-ole sprockets (4.2.1) concurrent-ruby (~> 1.0) @@ -460,7 +466,7 @@ GEM tca_client (1.0.4) typhoeus (~> 1.0, >= 1.0.1) tcp_timeout (0.1.1) - thor (1.3.0) + thor (1.3.1) tilt (2.3.0) timeout (0.4.1) ttfunk (1.8.0) @@ -470,9 +476,10 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.5.0) + uri (0.13.0) warden (1.2.9) rack (>= 2.0.9) - webmock (3.19.1) + webmock (3.23.0) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) @@ -480,8 +487,8 @@ GEM websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - yard (0.9.34) - zeitwerk (2.6.12) + yard (0.9.36) + zeitwerk (2.6.13) PLATFORMS aarch64-linux From 6462ccb7e716952600a187733c4ca65b3e35b12b Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 20:34:46 +1000 Subject: [PATCH 12/27] chore(release): 8.0.0-6 --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fe33fc0..63ab48cc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-6](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-5...v8.0.0-6) (2024-05-02) + + +### Features + +* add the pdf-reader gem for validating pdf submissions ([71c845b](https://github.com/macite/doubtfire-deploy/commit/71c845bf28fccf28de17ed83e3da1cf243646e7b)) +* implement unit test for pdf validation on submit ([57db1dc](https://github.com/macite/doubtfire-deploy/commit/57db1dc57a75aaf211030ecf78b9252a5d8b583b)) +* improve pdf file validation and detect encrypted pdfs ([dd729cf](https://github.com/macite/doubtfire-deploy/commit/dd729cf31bec115bd0e7018f33a692bd35bb5519)) + + +### Bug Fixes + +* add redis to the github actions workflow ([9935720](https://github.com/macite/doubtfire-deploy/commit/99357205d42d148f3a6165a96122691680409092)) +* document two new env variables for redis ([749903f](https://github.com/macite/doubtfire-deploy/commit/749903f390a388fac2c2a8652975580611f1e072)) +* implement error reporting in database populator ([136b9f9](https://github.com/macite/doubtfire-deploy/commit/136b9f98151688d3d6a578db1f980b39b3e21514)) +* install ruby-lsp in the development environment ([c57290e](https://github.com/macite/doubtfire-deploy/commit/c57290e4b2f7ba1bab7d600965988489dc3dd5a4)) +* pick up redis url from env for sidekiq if present ([e9628eb](https://github.com/macite/doubtfire-deploy/commit/e9628eb31398719d78508a610a251a785f56a14f)) +* revert to doubtfire local image for unit tests ([73fcbe3](https://github.com/macite/doubtfire-deploy/commit/73fcbe3f5adb603253033e7b126502a5d3c006f1)) +* skip unit tests and linting for documentation updates ([2503fe6](https://github.com/macite/doubtfire-deploy/commit/2503fe61468f8ebe37e54ccb1d0cc2a11387949b)) + +### [7.0.23](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-2...v7.0.23) (2024-03-22) + ## [8.0.0-5](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-4...v8.0.0-5) (2024-05-02) From f3f7028d7676e9925ca23d39ea9c2ca62ad3a4bf Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 20:49:02 +1000 Subject: [PATCH 13/27] chore: fix deployment bocker builder version --- .github/workflows/deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 2ca1d378c..c9461b31e 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -50,7 +50,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v5 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 if: github.event_name != 'pull_request' From 3e1b3fe2e445c78c36c3dc0812b0932620d92098 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 20:53:22 +1000 Subject: [PATCH 14/27] chore(release): 8.0.0-7 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ab48cc7..9f4084e0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-7](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-6...v8.0.0-7) (2024-05-02) + ## [8.0.0-6](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-5...v8.0.0-6) (2024-05-02) From ef8611f917b198064a891f81c02408ff081e977b Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 2 May 2024 23:19:51 +1000 Subject: [PATCH 15/27] fix: update for dotenv 3 --- Gemfile | 2 +- Gemfile.lock | 5 +---- config/application.rb | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index b937d0fa8..dd6d7e999 100644 --- a/Gemfile +++ b/Gemfile @@ -79,7 +79,7 @@ gem 'grape-swagger-rails' # Miscellaneous gem 'bunny-pub-sub', '0.5.2' gem 'ci_reporter' -gem 'dotenv-rails' +gem 'dotenv' gem 'rack-cors', require: 'rack/cors' gem 'require_all', '>=1.3.3' diff --git a/Gemfile.lock b/Gemfile.lock index 3bd5a04f4..06c3f1fcc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -131,9 +131,6 @@ GEM docile (1.4.0) domain_name (0.6.20240107) dotenv (3.1.1) - dotenv-rails (3.1.1) - dotenv (= 3.1.1) - railties (>= 6.1) drb (2.2.1) dry-core (1.0.1) concurrent-ruby (~> 1.0) @@ -504,7 +501,7 @@ DEPENDENCIES database_cleaner-active_record devise devise_ldap_authenticatable - dotenv-rails + dotenv factory_bot factory_bot_rails faker diff --git a/config/application.rb b/config/application.rb index c6a5ce9aa..beb541e27 100644 --- a/config/application.rb +++ b/config/application.rb @@ -17,7 +17,7 @@ class Application < Rails::Application config.load_defaults 7.0 # Load .env variables - Dotenv::Railtie.load + Dotenv::Rails.load # ==> Authentication Method # Authentication method default is database, but possible settings From 4bfdfb1faf5bbaf45ec7827ec89e4cd231f88dba Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Sat, 11 May 2024 15:03:27 +1000 Subject: [PATCH 16/27] fix: adjust log and params in tii_actions --- app/models/turn_it_in/tii_action.rb | 4 ++-- db/migrate/20240105055902_add_tii_details.rb | 4 ++-- db/schema.rb | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/models/turn_it_in/tii_action.rb b/app/models/turn_it_in/tii_action.rb index f4692ec77..190792e27 100644 --- a/app/models/turn_it_in/tii_action.rb +++ b/app/models/turn_it_in/tii_action.rb @@ -49,12 +49,12 @@ def perform self.error_code = nil if self.retry && error? self.custom_error_message = nil - self.log = [] if self.complete # reset log if complete... and performing again + self.log = [] if self.log.nil? || self.log.empty? || self.complete # reset log if complete... and performing again self.log << { date: Time.zone.now, message: "Started #{type}" } self.last_run = Time.zone.now + self.retry = false # reset retry flag - self.log = [] if self.complete # reset log if complete... and performing again self.complete = false # reset complete flag result = run diff --git a/db/migrate/20240105055902_add_tii_details.rb b/db/migrate/20240105055902_add_tii_details.rb index 42a0bee33..a571f80d1 100644 --- a/db/migrate/20240105055902_add_tii_details.rb +++ b/db/migrate/20240105055902_add_tii_details.rb @@ -67,8 +67,8 @@ def change t.integer :error_code t.text :custom_error_message - t.text :log, default: "[]" - t.text :params, default: "{}" + t.text :log + t.string :params, default: "{}", limit: 1024 t.timestamps diff --git a/db/schema.rb b/db/schema.rb index 645806173..3a6368943 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -355,8 +355,8 @@ t.boolean "retry", default: true, null: false t.integer "error_code" t.text "custom_error_message" - t.text "log", default: "[]" - t.text "params", default: "{}" + t.text "log" + t.string "params", limit: 1024, default: "{}" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["complete"], name: "index_tii_actions_on_complete" From a1f72b1d825891e9888b7068e32673df9a950900 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Sat, 11 May 2024 15:20:36 +1000 Subject: [PATCH 17/27] chore(release): 8.0.0-8 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f4084e0d..99a677579 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-8](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-7...v8.0.0-8) (2024-05-11) + + +### Bug Fixes + +* adjust log and params in tii_actions ([4bfdfb1](https://github.com/macite/doubtfire-deploy/commit/4bfdfb1faf5bbaf45ec7827ec89e4cd231f88dba)) +* update for dotenv 3 ([ef8611f](https://github.com/macite/doubtfire-deploy/commit/ef8611f917b198064a891f81c02408ff081e977b)) + ## [8.0.0-7](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-6...v8.0.0-7) (2024-05-02) ## [8.0.0-6](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-5...v8.0.0-6) (2024-05-02) From c3a22bfee6e9912fd8b4d331d6a6e6f350b72ffa Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Sat, 11 May 2024 16:17:33 +1000 Subject: [PATCH 18/27] fix: revert to default cache store --- config/environments/production.rb | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 9e7018c65..6bb6af878 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -2,18 +2,6 @@ Doubtfire::Application.configure do # Code is not reloaded between requests config.cache_classes = true - config.cache_store = :redis_cache_store, { url: ENV.fetch('DF_REDIS_CACHE_URL', 'redis://localhost:6379/0'), - - connect_timeout: 30, # Defaults to 1 second - read_timeout: 0.2, # Defaults to 1 second - write_timeout: 0.2, # Defaults to 1 second - reconnect_attempts: 2, # Defaults to 1 - - error_handler: lambda { |method:, returning:, exception:| - # Report errors to Sentry as warnings - Sentry.capture_exception exception, level: 'warning', - tags: { method: method, returning: returning } - } } # Full error reports are disabled and caching is turned on config.consider_all_requests_local = false From a9959fef2223ffca41338b15c973b888253225bf Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Sat, 11 May 2024 19:44:51 +1000 Subject: [PATCH 19/27] fix: ensure default log in tii actions --- app/models/turn_it_in/tii_action.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/turn_it_in/tii_action.rb b/app/models/turn_it_in/tii_action.rb index 190792e27..d20d5c0d6 100644 --- a/app/models/turn_it_in/tii_action.rb +++ b/app/models/turn_it_in/tii_action.rb @@ -33,6 +33,7 @@ class TiiAction < ApplicationRecord serialize :params, coder: JSON serialize :log, coder: JSON + attribute :log, default: -> { [] } def description 'Generic Turnitin Action' From d7c9c3c8c60b49721aa9cac1f8df6ec716b82422 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Sat, 11 May 2024 19:47:32 +1000 Subject: [PATCH 20/27] fix: ensure tii launch handles errors so rails can progress --- app/helpers/turn_it_in.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/helpers/turn_it_in.rb b/app/helpers/turn_it_in.rb index 57aefc0f1..9f4bd96af 100644 --- a/app/helpers/turn_it_in.rb +++ b/app/helpers/turn_it_in.rb @@ -41,6 +41,8 @@ def self.launch_tii(with_webhooks: true) TiiRegisterWebHookJob.perform_async if with_webhooks load_tii_features load_tii_eula + rescue StandardError => e + Rails.logger.error "Error launching TII: #{e}" end # Check if the features are up to date, and update if required From f4ac39b6d52dc44604fdbc8a78351b995e40966a Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Sat, 11 May 2024 19:47:46 +1000 Subject: [PATCH 21/27] chore(release): 8.0.0-9 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99a677579..06523f941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-9](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-8...v8.0.0-9) (2024-05-11) + + +### Bug Fixes + +* ensure default log in tii actions ([a9959fe](https://github.com/macite/doubtfire-deploy/commit/a9959fef2223ffca41338b15c973b888253225bf)) +* ensure tii launch handles errors so rails can progress ([d7c9c3c](https://github.com/macite/doubtfire-deploy/commit/d7c9c3c8c60b49721aa9cac1f8df6ec716b82422)) +* revert to default cache store ([c3a22bf](https://github.com/macite/doubtfire-deploy/commit/c3a22bfee6e9912fd8b4d331d6a6e6f350b72ffa)) + ## [8.0.0-8](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-7...v8.0.0-8) (2024-05-11) From 3cd67d7c58916cda429d3c0266942cfc2c0ef878 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Mon, 13 May 2024 14:26:52 +1000 Subject: [PATCH 22/27] fix: host url for turn it in integration --- app/api/authentication_api.rb | 16 ++++++++++------ app/helpers/turn_it_in.rb | 2 +- app/mailers/notifications_mailer.rb | 2 +- app/mailers/portfolio_evidence_mailer.rb | 2 +- config/application.rb | 5 ++--- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/app/api/authentication_api.rb b/app/api/authentication_api.rb index bc9b09542..3b43b10cf 100644 --- a/app/api/authentication_api.rb +++ b/app/api/authentication_api.rb @@ -141,9 +141,11 @@ class AuthenticationApi < Grape::API logger.info "Redirecting #{user.username} from #{request.ip}" # Must redirect to the front-end after sign in - protocol = Rails.env.development? ? 'http' : 'https' - host = Rails.env.development? ? "#{protocol}://localhost:3000" : Doubtfire::Application.config.institution[:host] - host = "#{protocol}://#{host}" unless host.starts_with?('http') + host = Doubtfire::Application.config.institution[:host] + unless host.starts_with?('http') + protocol = Rails.env.development? ? 'http' : 'https' + host = "#{protocol}://#{host}" + end redirect "#{host}/#/sign_in?authToken=#{onetime_token.authentication_token}&username=#{user.username}" end end @@ -216,9 +218,11 @@ class AuthenticationApi < Grape::API logger.info "Redirecting #{user.username} from #{request.ip}" # Must redirect to the front-end after sign in - protocol = Rails.env.development? ? 'http' : 'https' - host = Rails.env.development? ? "#{protocol}://localhost:3000" : Doubtfire::Application.config.institution[:host] - host = "#{protocol}://#{host}" unless host.starts_with?('http') + host = Doubtfire::Application.config.institution[:host] + unless host.starts_with?('http') + protocol = Rails.env.development? ? 'http' : 'https' + host = "#{protocol}://#{host}" + end redirect "#{host}/#/sign_in?authToken=#{onetime_token.authentication_token}&username=#{user.username}" end end diff --git a/app/helpers/turn_it_in.rb b/app/helpers/turn_it_in.rb index 9f4bd96af..b77bcb53f 100644 --- a/app/helpers/turn_it_in.rb +++ b/app/helpers/turn_it_in.rb @@ -152,7 +152,7 @@ def self.load_tii_eula # Return the url used for webhook callbacks def self.webhook_url - "#{Doubtfire::Application.config.institution[:host_url]}api/tii_hook" + "#{Doubtfire::Application.config.institution[:host]}/api/tii_hook" end # Create or get the group context for a unit. The "group context" is the Turn It In equivalent of a unit. diff --git a/app/mailers/notifications_mailer.rb b/app/mailers/notifications_mailer.rb index 767e3d53d..741fa18b8 100644 --- a/app/mailers/notifications_mailer.rb +++ b/app/mailers/notifications_mailer.rb @@ -2,7 +2,7 @@ class NotificationsMailer < ActionMailer::Base def add_general @doubtfire_host = Doubtfire::Application.config.institution[:host] @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] - @unsubscribe_url = "https://#{@doubtfire_host}/#/home?notifications" + @unsubscribe_url = "#{@doubtfire_host}/#/home?notifications" end def weekly_staff_summary(unit_role, summary_stats) diff --git a/app/mailers/portfolio_evidence_mailer.rb b/app/mailers/portfolio_evidence_mailer.rb index b2e255ab2..61b729eb6 100644 --- a/app/mailers/portfolio_evidence_mailer.rb +++ b/app/mailers/portfolio_evidence_mailer.rb @@ -2,7 +2,7 @@ class PortfolioEvidenceMailer < ActionMailer::Base def add_general @doubtfire_host = Doubtfire::Application.config.institution[:host] @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] - @unsubscribe_url = "https://#{@doubtfire_host}/#/home?notifications" + @unsubscribe_url = "#{@doubtfire_host}/#/home?notifications" end def task_pdf_failed(project, tasks) diff --git a/config/application.rb b/config/application.rb index beb541e27..fa4bcb709 100644 --- a/config/application.rb +++ b/config/application.rb @@ -47,9 +47,8 @@ class Application < Rails::Application config.institution[:product_name] = ENV['DF_INSTITUTION_PRODUCT_NAME'] if ENV['DF_INSTITUTION_PRODUCT_NAME'] config.institution[:privacy] = ENV['DF_INSTITUTION_PRIVACY'] if ENV['DF_INSTITUTION_PRIVACY'] config.institution[:plagiarism] = ENV['DF_INSTITUTION_PLAGIARISM'] if ENV['DF_INSTITUTION_PLAGIARISM'] - # Institution host becomes localhost in all but prod - config.institution[:host] = 'localhost:3000' if Rails.env.development? - config.institution[:host_url] = Rails.env.development? ? "http://#{config.institution[:host]}/" : "https://#{config.institution[:host]}/" + # Institution host becomes localhost in development + config.institution[:host] |= 'http://localhost:3000' if Rails.env.development? config.institution[:settings] = ENV['DF_INSTITUTION_SETTINGS_RB'] if ENV['DF_INSTITUTION_SETTINGS_RB'] config.institution[:ffmpeg] = ENV['DF_FFMPEG_PATH'] || 'ffmpeg' From 8d7e94bf71c1349a0c98370be52face89a723c9b Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Mon, 13 May 2024 14:28:51 +1000 Subject: [PATCH 23/27] chore(release): 8.0.0-10 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06523f941..c2d3f8836 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-10](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-9...v8.0.0-10) (2024-05-13) + + +### Bug Fixes + +* host url for turn it in integration ([3cd67d7](https://github.com/macite/doubtfire-deploy/commit/3cd67d7c58916cda429d3c0266942cfc2c0ef878)) + ## [8.0.0-9](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-8...v8.0.0-9) (2024-05-11) From 99e370867e265fc213b4159f5d4d856225376b49 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Mon, 13 May 2024 17:13:06 +1000 Subject: [PATCH 24/27] Ensure that register webhooks registers complete --- app/models/turn_it_in/tii_action_register_webhook.rb | 1 + config/application.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/turn_it_in/tii_action_register_webhook.rb b/app/models/turn_it_in/tii_action_register_webhook.rb index f9a44c97b..b004125f6 100644 --- a/app/models/turn_it_in/tii_action_register_webhook.rb +++ b/app/models/turn_it_in/tii_action_register_webhook.rb @@ -10,6 +10,7 @@ def description def run register_webhook if need_to_register_webhook? + self.complete = true end def need_to_register_webhook? diff --git a/config/application.rb b/config/application.rb index fa4bcb709..d2449dd24 100644 --- a/config/application.rb +++ b/config/application.rb @@ -48,7 +48,7 @@ class Application < Rails::Application config.institution[:privacy] = ENV['DF_INSTITUTION_PRIVACY'] if ENV['DF_INSTITUTION_PRIVACY'] config.institution[:plagiarism] = ENV['DF_INSTITUTION_PLAGIARISM'] if ENV['DF_INSTITUTION_PLAGIARISM'] # Institution host becomes localhost in development - config.institution[:host] |= 'http://localhost:3000' if Rails.env.development? + config.institution[:host] ||= 'http://localhost:3000' if Rails.env.development? config.institution[:settings] = ENV['DF_INSTITUTION_SETTINGS_RB'] if ENV['DF_INSTITUTION_SETTINGS_RB'] config.institution[:ffmpeg] = ENV['DF_FFMPEG_PATH'] || 'ffmpeg' From 1d208ca58d4f317d9c94cbb6a42308518b81d497 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Mon, 13 May 2024 17:13:20 +1000 Subject: [PATCH 25/27] chore(release): 8.0.0-11 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2d3f8836..ac464e585 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0-11](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-10...v8.0.0-11) (2024-05-13) + ## [8.0.0-10](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-9...v8.0.0-10) (2024-05-13) From f808ad437f424f40a4eb68d5218ddf4317ba44b6 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Wed, 22 May 2024 11:56:16 +1000 Subject: [PATCH 26/27] fix: correct typos in unit role needed for teaching role --- app/models/unit_role.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/unit_role.rb b/app/models/unit_role.rb index bde2d104f..9985a5174 100644 --- a/app/models/unit_role.rb +++ b/app/models/unit_role.rb @@ -150,9 +150,9 @@ def send_weekly_status_email(summary_stats) def ensure_valid_user_for_role if is_convenor? - errors.add :user, 'must have a role that id able to administer units (request admin to adjust user role)' unless user.has_convenor_capability? + errors.add :user, 'must have a role that is able to administer units (request admin to adjust user role)' unless user.has_convenor_capability? else - errors.add :user, 'must have a role that id able to teach units (request admin to adjust user role)' unless user.has_tutor_capability? + errors.add :user, 'must have a role that is able to teach units (request admin to adjust user role)' unless user.has_tutor_capability? end end From 5cdff98e54379fff03e30b6d3edb44d962575bb9 Mon Sep 17 00:00:00 2001 From: Andrew Cain Date: Thu, 23 May 2024 23:37:21 +1000 Subject: [PATCH 27/27] chore(release): 8.0.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac464e585..e9b443de9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.0.0](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-11...v8.0.0) (2024-05-23) + + +### Bug Fixes + +* correct typos in unit role needed for teaching role ([f808ad4](https://github.com/macite/doubtfire-deploy/commit/f808ad437f424f40a4eb68d5218ddf4317ba44b6)) + ## [8.0.0-11](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-10...v8.0.0-11) (2024-05-13) ## [8.0.0-10](https://github.com/macite/doubtfire-deploy/compare/v8.0.0-9...v8.0.0-10) (2024-05-13)