From 078e1f54b71a105fa6f6ffc6cf55e66d8b0c9143 Mon Sep 17 00:00:00 2001 From: Pei Chi Huang Date: Wed, 20 Dec 2023 16:04:44 +0100 Subject: [PATCH] bagit upload download enhancement --- lib/labimotion.rb | 2 +- lib/labimotion/apis/labimotion_hub_api.rb | 2 +- lib/labimotion/libs/converter.rb | 5 ++- lib/labimotion/libs/export_dataset.rb | 50 +++++++++++++++++------ lib/labimotion/libs/nmr_mapper.rb | 18 +++++--- lib/labimotion/version.rb | 6 +-- 6 files changed, 57 insertions(+), 26 deletions(-) diff --git a/lib/labimotion.rb b/lib/labimotion.rb index 104079d..0d05d77 100644 --- a/lib/labimotion.rb +++ b/lib/labimotion.rb @@ -8,7 +8,7 @@ def self.logger end def self.log_exception(exception, current_user = nil) - Labimotion.logger.error("version: #{Labimotion::VERSION}; #{Labimotion::IS_RAILS5}, (#{current_user&.id}) \n Exception: #{exception.message}") + Labimotion.logger.error("version: #{Labimotion::VERSION}; (#{current_user&.id}) \n Exception: #{exception.message}") Labimotion.logger.error(exception.backtrace.join("\n")) end diff --git a/lib/labimotion/apis/labimotion_hub_api.rb b/lib/labimotion/apis/labimotion_hub_api.rb index aa96610..9abf66e 100644 --- a/lib/labimotion/apis/labimotion_hub_api.rb +++ b/lib/labimotion/apis/labimotion_hub_api.rb @@ -19,7 +19,7 @@ class LabimotionHubAPI < Grape::API get do list = "Labimotion::#{params[:klass]}".constantize.where(is_active: true).where.not(released_at: nil) list = list.where(is_generic: true) if params[:klass] == 'ElementKlass' - entities = Labimotion::GenericPublicEntity.represent(list, displayed: params[:with_props]) + entities = Labimotion::GenericPublicEntity.represent(list, displayed: params[:with_props], root: 'list') rescue StandardError => e Labimotion.log_exception(e, current_user) [] diff --git a/lib/labimotion/libs/converter.rb b/lib/labimotion/libs/converter.rb index 0c08888..cb3b1a9 100644 --- a/lib/labimotion/libs/converter.rb +++ b/lib/labimotion/libs/converter.rb @@ -20,7 +20,10 @@ def self.logger def self.process_ds(id, current_user = {}) att = Attachment.find_by(id: id, con_state: Labimotion::ConState::CONVERTED) - return if att.nil? || att.attachable_id.nil? || att.attachable_type != 'Container' + return if att.nil? || att.attachable_id.nil? || att.attachable_type != 'Container' || att.filename.split('.')&.last != 'zip' + + eln_ds = Container.find_by(id: att.attachable_id, container_type: 'dataset') + return if eln_ds.nil? || eln_ds.parent.nil? || eln_ds.parent&.container_type != 'analysis' dsr = [] ols = nil diff --git a/lib/labimotion/libs/export_dataset.rb b/lib/labimotion/libs/export_dataset.rb index 50d481e..9ccfae0 100644 --- a/lib/labimotion/libs/export_dataset.rb +++ b/lib/labimotion/libs/export_dataset.rb @@ -40,6 +40,8 @@ def description(ds, id) header_style = sheet.styles.add_style(sz: 12, fg_color: 'FFFFFF', bg_color: '00008B', border: { style: :thick, color: 'FF777777', edges: [:bottom] }) sheet.add_row(['File name', res_name(id)]) sheet.add_row(['Time', Time.now.strftime("%Y-%m-%d %H:%M:%S %Z")] ) + sheet.add_row(['(This file is automatically generated by the system.)']) + sheet.add_row(['']) sheet.add_row(['']) sheet.add_row(['Fields description of sheet:' + ds.dataset_klass.label]) sheet.add_row(['Fields', 'Field description'], style: header_style) @@ -49,13 +51,11 @@ def description(ds, id) sheet.add_row(['Unit', 'The unit of the field']) sheet.add_row(['Name', 'The key of the field, can be used to identify the field']) sheet.add_row(['Type', 'The type of the field']) - sheet.add_row(['From device?', '[Yes] if the field is from the device, [No] if the field is manually entered by the user, [SYS] if the field is automatically generated by the system']) - sheet.add_row(['Device source', 'The source tag of the device file, available only if the ontology term is 1H NMR or 13C NMR']) - sheet.add_row(['Device data', 'The original device data, can not be changed after data uploaded']) - sheet.add_row(['']) + sheet.add_row(['Source?', '[Device] from device, [Chemotion] from Chemotion']) + sheet.add_row(['Source identifier', 'The source identifier']) + sheet.add_row(['Source data', 'The data from Device or Chemotion, cannot be modified once a generic dataset is created']) sheet.add_row(['']) sheet.add_row(['']) - sheet.add_row(['', '(This file is automatically generated by the system.)']) end def export(id) @@ -84,29 +84,41 @@ def export(id) next if field['type'] == 'dummy' type = field['type'] - from_device = field['device'].present? ? 'Yes' : 'No' - from_device = field['system'].present? ? 'SYS' : from_device + from_device = field['device'].present? ? 'Device' : '' + from_device = field['system'].present? ? 'Chemotion' : from_device type = "#{field['type']}-#{field['option_layers']}" if field['type'] == 'select' || field['type'] == 'system-defined' show_value = field['value'] =~ /\A\d+,\d+\z/ ? field['value']&.gsub(',', '.') : field['value'] - sheet.add_row([' ', field['label'], show_value, field['value_system'], field['field'], type, from_device, field['dkey'], field['system'] || field['device']].freeze) + sheet.add_row([' ', field['label'], nil, field['value_system'], field['field'], type, from_device, field['dkey'], nil].freeze) + + if %w[system-defined integer].include? field['type'] + sheet.rows.last.cells[2].type = :integer + sheet.rows.last.cells[8].type = :integer + else + sheet.rows.last.cells[2].type = :string + sheet.rows.last.cells[8].type = :string + end + sheet.rows.last.cells[2].value = show_value + sheet.rows.last.cells[8].value = field['system'] || field['device'] + + end # sheet.column_widths nil, nil, nil, nil, 0, 0, 0, 0, 0 end end def spectra(id) + name_mapping = [] wb = @xfile.workbook gds = Labimotion::Dataset.find_by(element_id: id, element_type: 'Container') cds = Container.find(id) - cds_csv = cds.attachments.where(aasm_state: 'csv') + cds_csv = cds.attachments.where(aasm_state: 'csv').order(:filename) csv_length = cds_csv.length return if csv_length.zero? cds_csv.each_with_index do |att, idx| - name = File.basename(att.filename, '.csv') - name = name.slice(0, (25 - csv_length.to_s.length - 1)) - sheet_name = "#{name}_#{idx}" + sheet_name = "Sheet#{idx+1}" sheet = @xfile.workbook.add_worksheet(name: sheet_name) + name_mapping.push([sheet_name, att.filename]) if Labimotion::IS_RAILS5 == true File.open(att.store.path) do |fi| @@ -122,10 +134,22 @@ def spectra(id) end end end + + if name_mapping.length > 1 + first_sheet = @xfile.workbook.worksheets&.first + header_style = first_sheet&.styles.add_style(sz: 12, fg_color: 'FFFFFF', bg_color: '00008B', border: { style: :thick, color: 'FF777777', edges: [:bottom] }) + first_sheet&.add_row(['Sheet name', 'File name'], style: header_style) + name_mapping&.each do |mapping| + next if mapping.length < 2 + + @xfile.workbook.worksheets&.first&.add_row([mapping[0].to_s, mapping[1].to_s]) + end + end + end def header - ['Layer Label', 'Field Label', 'Value', 'Unit', 'Name', 'Type', 'from device?', 'Device source', 'Device data'].freeze + ['Layer Label', 'Field Label', 'Value', 'Unit', 'Name', 'Type', 'Source?', 'Source identifier', 'Source data'].freeze end def read diff --git a/lib/labimotion/libs/nmr_mapper.rb b/lib/labimotion/libs/nmr_mapper.rb index 7398d69..e7c718f 100644 --- a/lib/labimotion/libs/nmr_mapper.rb +++ b/lib/labimotion/libs/nmr_mapper.rb @@ -10,8 +10,12 @@ def self.process_ds(id, current_user = {}) att = Attachment.find_by(id: id, con_state: Labimotion::ConState::NMR) return if att.nil? - content = is_brucker_binary(id) - if content.nil? + result = is_brucker_binary(id) + if result[:is_bagit] == true + att.update_column(:con_state, Labimotion::ConState::CONVERTED) + Labimotion::Converter.metadata(id) + Labimotion::ConState::COMPLETED + elsif result[:metadata] == nil Labimotion::ConState::NONE else data = process(att, id, content) @@ -29,7 +33,9 @@ def self.is_brucker_binary(id) zip_file.each do |entry| if entry.name.include?('/pdata/') && entry.name.include?('parm.txt') metadata = entry.get_input_stream.read.force_encoding('UTF-8') - return metadata + return { is_bagit: false, metadata: metadata } + elsif entry.name.include?('metadata/') && entry.name.include?('converter.json') + return { is_bagit: true, metadata: nil } end end end @@ -39,13 +45,15 @@ def self.is_brucker_binary(id) zip_file.each do |entry| if entry.name.include?('/pdata/') && entry.name.include?('parm.txt') metadata = entry.get_input_stream.read.force_encoding('UTF-8') - return metadata + return { is_bagit: false, metadata: metadata } + elsif entry.name.include?('metadata/') && entry.name.include?('converter.json') + return { is_bagit: true, metadata: nil } end end end end end - nil + { is_bagit: false, metadata: nil } end def self.process(att, id, content) diff --git a/lib/labimotion/version.rb b/lib/labimotion/version.rb index 9e453a2..f76f7e7 100644 --- a/lib/labimotion/version.rb +++ b/lib/labimotion/version.rb @@ -3,9 +3,5 @@ ## Labimotion Version module Labimotion IS_RAILS5 = false - VERSION_ELN = '1.0.19' - VERSION_REPO = '0.3.1' - - VERSION = Labimotion::VERSION_REPO if Labimotion::IS_RAILS5 == true - VERSION = Labimotion::VERSION_ELN if Labimotion::IS_RAILS5 == false + VERSION = '1.1.0' end