Merge main

Gemfile.lock

@@ -243,8 +243,7 @@ GEM
     regexp_parser (2.9.0)
     reline (0.5.10)
       io-console (~> 0.5)
-    rexml (3.3.6)
-      strscan
+    rexml (3.3.9)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -298,7 +297,6 @@ GEM
     stimulus-rails (1.3.4)
       railties (>= 6.0.0)
     stringio (3.1.1)
-    strscan (3.1.0)
     thor (1.3.2)
     timeout (0.4.1)
     timers (4.3.5)

app/controllers/annotations_controller.rb

@@ -1,20 +1,20 @@
 class AnnotationsController < ApplicationController
 
   def create
-    @response = SurveyResponse.find(sanitized_params[:survey_response_id])
-    @annotation = Annotation.new(survey_response_id: @response.id, text: sanitized_params[:text])
+    @survey_response = SurveyResponse.find(sanitized_params[:survey_response_id])
+    @annotation = Annotation.new(survey_response_id: @survey_response.id, text: sanitized_params[:text])
     success = @annotation.save
 
     respond_to do |format|
       format.turbo_stream do
-        render turbo_stream: turbo_stream.replace("annotation", partial: "/annotations/form", locals: { response: @response, annotation: @annotation, success: success  })
+        render turbo_stream: turbo_stream.replace("annotation", partial: "/annotations/form", locals: { response: @survey_response, annotation: @annotation, success: success  })
       end
     end
   end
 
   def update
-    @response = SurveyResponse.find(sanitized_params[:survey_response_id])
-    @annotation = Annotation.find_or_initialize_by(survey_response_id: @response.id)
+    @survey_response = SurveyResponse.find(sanitized_params[:survey_response_id])
+    @annotation = Annotation.find_or_initialize_by(survey_response_id: @survey_response.id)
     @annotation.text = sanitized_params[:text]
     if sanitized_params[:text].empty?
       success = @annotation.destroy
@@ -24,7 +24,7 @@ def update
 
     respond_to do |format|
       format.turbo_stream do
-        render turbo_stream: turbo_stream.replace("annotation", partial: "/annotations/form", locals: { response: @response, annotation: @annotation, success: success  })
+        render turbo_stream: turbo_stream.replace("annotation", partial: "/annotations/form", locals: { response: @survey_response, annotation: @annotation, success: success  })
       end
     end
   end

app/controllers/codebooks_controller.rb

@@ -1,37 +1,36 @@
 class CodebooksController < ApplicationController
 
   def index
-    @contexts = Question::QUESTIONS
+    @questions = Question.all.order(:created_at)
   end
 
   def show
-    @context = params[:id].gsub("class", "klass")
-    @question = Question.from(@context)
-    @context_key = @question.context
+    @question = Question.find(params[:id])
+    @context = @question.context
+    @enqueued_at = params[:enqueued_at].present? ? Time.at(params[:enqueued_at].to_i).strftime("%T %Z") : nil
 
     # Support the previous/next navigation controls
 
-    sections = Question::QUESTIONS.keys.map(&:to_s)
-    previous_index = (sections.index(@question.key) - 1)
-    next_index = (sections.index(@question.key) + 1) % sections.length
+    sections = Question.all.pluck(:id)
+    previous_index = (sections.index(@question.id) - 1)
+    next_index = (sections.index(@question.id) + 1) % sections.length
     @section_name = @question.label
     @previous_section = sections[previous_index]
     @next_section = sections[next_index]
 
-    if @question.identity_field?
+    if @question.is_identity?
       # Identity fields have associated Identity objects.
-      @frequencies = Identity.histogram(@context)
+      @frequencies = Identity.histogram(@context.name)
       @frequencies_by_keys = @frequencies.sort{|a, b| a[0] <=> b[0]}
       @frequencies_by_values = @frequencies.sort{|a, b| a[1] <=> b[1]}
-    elsif @question.experience_field?
+    else
       # Experience fields have associated Code and Category objects.
-      @frequencies = Code.histogram(@context)
+      @frequencies = Code.histogram(@context.name)
       @frequencies_by_keys = @frequencies.sort{|a, b| a[0] <=> b[0]}
       @frequencies_by_values = @frequencies.sort{|a, b| a[1] <=> b[1]}
-      @categories_histogram = Category.histogram(@context)
+      @categories_histogram = Category.histogram(@context.name)
       @total_codes = @categories_histogram.values.sum
-      Rails.logger.info("!!! params[:id] = #{params[:id]}")
-      @codes = Code.where(context: params[:id].gsub("klass", "class").gsub("_exp", "").gsub("_", "-"))
+      @codes = Code.where(context: @context.name)
     end
 
   end

app/controllers/questions_controller.rb

@@ -1,8 +1,8 @@
 class QuestionsController < ApplicationController
 
   def show
-    @question = Question.from(params[:id])
-    @responses = SurveyResponse.where("#{@question.key} IS NOT NULL").order(:created_at)
+    @question = Question.find(params[:id])
+    @responses = @question.responses.order(:survey_response_id)
   end
 
 end

app/controllers/responses_controller.rb

@@ -0,0 +1,20 @@
+class ResponsesController < ApplicationController
+
+  def update
+    @response = Response.find(params[:id])
+    success = @response.update(raw_codes: response_params[:raw_codes].join(",").split(",").reject(&:empty?).compact.map(&:strip).map(&:downcase))
+
+    respond_to do |format|
+      format.turbo_stream do
+        render turbo_stream: turbo_stream.replace("frame-response-#{@response.id}", partial: "/responses/form", locals: { response: @response, success: success, filters: false  })
+      end
+    end
+  end
+
+  private
+
+  def response_params
+    params.require(:response).permit(raw_codes: [])
+  end
+
+end

app/controllers/stats_controller.rb

@@ -3,7 +3,7 @@ class StatsController < ApplicationController
   def index
 
     survey_response_count = SurveyResponse.count
-    question_count = Question::QUESTIONS.count
+    question_count = Question.all.count
     answer_count = survey_response_count * question_count
     persona_count = Persona.count
     identity_count = Identity.count

app/controllers/survey_responses_controller.rb

@@ -1,20 +1,21 @@
 class SurveyResponsesController < ApplicationController
 
   def index
-    @responses = SurveyResponse.all.order(:created_at)
+    @survey_responses = SurveyResponse.all.order(:created_at)
   end
 
   def show
-    @response = SurveyResponse.find(params[:id])
+    @survey_response = SurveyResponse.find(params[:id])
     @enqueued_at = params[:enqueued_at]
 
-    @previous_response = SurveyResponse.where("created_at < ?", @response.created_at).order("created_at DESC").limit(1).first
-    @next_response = SurveyResponse.where("created_at > ?", @response.created_at).order("created_at ASC").limit(1).first
+    @previous_response = SurveyResponse.where("created_at < ?", @survey_response.created_at).order("created_at DESC").limit(1).first
+    @next_response = SurveyResponse.where("created_at > ?", @survey_response.created_at).order("created_at ASC").limit(1).first
 
-    persona = Persona.find_or_initialize_by(survey_response_id: @response.id)
+    persona = Persona.find_or_initialize_by(survey_response_id: @survey_response.id)
     @categories = persona.categories.sort{ |a,b| "#{a.context}.#{a.name}" <=> "#{b.context}.#{b.name}" }
     @keywords = persona.keywords.order_by(:name)
-    @annotation = @response.annotation || Annotation.new(survey_response_id: @response.id)
+    @annotation = @survey_response.annotation || Annotation.new(survey_response_id: @survey_response.id)
+    @responses = @survey_response.responses.order(:created_at)
   end
 
   def new
@@ -25,32 +26,9 @@ def create
     redirect_to survey_responses_path
   end
 
-  def update
-    @response = SurveyResponse.find(params[:id])
-    sanitized_params = {}
-    response_params.each do |key, value|
-      sanitized_params[key] = value.join(",").split(",").reject(&:empty?).compact.map(&:strip).map(&:downcase)
-    end
-
-    success = @response.update(sanitized_params)
-    @question = Question.from(params[:survey_response].keys.first.gsub("_codes","").gsub("_id", "_given"))
-
-    respond_to do |format|
-      format.turbo_stream do
-        render turbo_stream: turbo_stream.replace("#{@question.key}_#{@response.id}", partial: "/survey_responses/form", locals: { response: @response, question: @question, success: success, filters: false  })
-      end
-    end
-  end
-
   def enqueue_keywords
     KeywordExtractorJob.perform_async(params[:survey_response_id])
     redirect_to( action: :show, id: params[:survey_response_id], params: {enqueued_at: Time.now.strftime("%I:%M:%S %P (%Z)")} )
   end
 
-  private
-
-    def response_params
-      params.require(:survey_response).permit(themes: [], age_exp_codes: [], klass_exp_codes: [], race_ethnicity_exp_codes: [], religion_exp_codes: [], disability_exp_codes: [], neurodiversity_exp_codes: [], gender_exp_codes: [], lgbtqia_exp_codes: [], age_id_codes: [], klass_id_codes: [], race_ethnicity_id_codes: [], religion_id_codes: [], disability_id_codes: [], neurodiversity_id_codes: [], gender_id_codes: [], lgbtqia_id_codes: [], pronouns_id_codes: [], pronouns_exp_codes: [], pronouns_feel_codes: [], affinity_codes: [], notes_codes: [])
-    end
-
 end

app/controllers/themes_controller.rb

@@ -9,7 +9,7 @@ def new
   end
 
   def show
-    @contexts = Theme::CONTEXTS
+    @contexts = Context.all.order(:name)
     @theme = Theme.find(params[:id])
     @categories = Category.all
   end

app/jobs/populate_survey_response_job.rb

@@ -0,0 +1,17 @@
+# This background job performs the Keyword extraction process.
+class PopulateSurveyResponseJob
+
+  include Sidekiq::Job
+
+  queue_as :default
+
+  def perform(context, record)
+    Rails.logger.info("PopulateSurveyResponseJob running with context #{context}")
+    survey_response = SurveyResponse.from(context, record)
+    survey_response.generate_wordcloud
+    survey_response.classify_sentiment
+    Keyword.from(self.id)
+  end
+
+end
+

app/models/category.rb

@@ -14,7 +14,7 @@ class Category
   validates :context, presence: true
 
   has_many :out, :codes, rel_class: :CategorizedAs
-  has_many :in, :categories, rel_class: :EmergesFrom
+  has_many :in, :themes, rel_class: :EmergesFrom
 
   # Regenerates Category objects based on codes within a given context.
   # This method uses the Clients::OpenAi client passing the codes as an argument to the prompt.
@@ -41,7 +41,6 @@ def self.from(context)
 
   # Generates a hash with the unique category name as the key and the count of its associated codes as a value.
   def self.histogram(context)
-    context = Question.from(context).context
     categories = where(context: context).query_as(:c).with('c, count{(c)-[:CATEGORIZED_AS]-(code:Code)} AS ct').where('ct > 0').return("c.name, ct").order('ct DESC')
     categories.inject({}) {|accumulator,category| accumulator[category.values[0]] ||= 0; accumulator[category.values[0]] += category.values[1]; accumulator}
   end

app/models/code.rb

@@ -19,7 +19,6 @@ class Code
 
   # Given a context, generates a hash with each unique Codes as a key and the counts of its uses as a value.
   def self.histogram(context)
-    context = Question.from(context).context
     codes = where(context: context).query_as(:c).with('c, count{(c)-[:EXPERIENCES]-()} AS ct').where('ct > 0').order('c DESC').return('c.name, ct')
     codes.inject({}) {|accumulator,code| accumulator[code.values[0]] ||= 0; accumulator[code.values[0]] += code.values[1]; accumulator}
   end

app/models/context.rb

@@ -0,0 +1,10 @@
+# A Context reflects a demographic category.
+class Context < ApplicationRecord
+
+  validates_presence_of :name
+  validates_presence_of :display_name
+  validates_uniqueness_of :name
+
+  has_many :questions
+
+end

app/models/identity.rb

@@ -14,7 +14,6 @@ class Identity
 
   # Generates a hash consisting of Identities and their number of occurrences.
   def self.histogram(context)
-    context = Question.from(context).context
     identities = where(context: context).query_as(:i).with('i, count{(i)-[:IDENTIFIES_WITH]-(p:Persona)} AS c').return('i.name, c').order('c DESC')
     identities.inject({}) {|accumulator,identity| accumulator[identity.values[0]] ||= 0; accumulator[identity.values[0]] += identity.values[1]; accumulator}
   end

app/models/keyword.rb

@@ -31,7 +31,9 @@ def self.from(survey_response_id)
     return unless survey_response = SurveyResponse.find(survey_response_id.to_i)
     return unless persona = Persona.find_by(survey_response_id: survey_response_id.to_i)
 
-    response = Clients::OpenAi.request("#{PROMPT} #{survey_response.notes}")
+    corpus = survey_response.reflections_corpus
+
+    response = Clients::OpenAi.request("#{PROMPT} #{corpus}")
     response['words'].compact.map(&:downcase).uniq.each do |word|
       if keyword = Keyword.find_or_create_by(name: word)
         ReflectsOn.create(from_node: persona, to_node: keyword )

app/models/question.rb

@@ -1,72 +1,21 @@
 # A Question is a representation of a survey question.
 # This class provides convenience methods for navigating question keys and labels, as well as selecting topical subsets of questions.
 # For now, Questions are hardcoded and not persisted.
-class Question
+class Question < ApplicationRecord
 
-  attr_accessor :key, :label
-
-  QUESTIONS = {
-    age_given: "Age",
-    age_exp: "Experience with Age",
-    klass_given: "Class",
-    klass_exp: "Experience with Class",
-    race_ethnicity_given: "Race/Ethnicity",
-    race_ethnicity_exp: "Experience with Race/Ethnicity",
-    religion_given: "Religion",
-    religion_exp: "Experience with Religion",
-    disability_given: "Disability",
-    disability_exp: "Experience with Disability",
-    neurodiversity_given: "Neurodiversity",
-    neurodiversity_exp: "Experience with Neurodiversity",
-    gender_given: "Gender",
-    gender_exp: "Experience with Gender",
-    lgbtqia_given: "LGBTQIA+ Status",
-    lgbtqia_exp: "Experience with LGBTQIA+",
-    pronouns_given: "Pronouns Given",
-    pronouns_exp: "Experience with Pronouns",
-    pronouns_feel: "Pronoun Feelings",
-    affinity: "Identity Affinities",
-    notes: "Identity Reflection"
-  }
-
-  def self.from(key)
-    new(key: key, label: QUESTIONS[key.to_sym])
-  end
+  has_many :responses
+  belongs_to :context
 
   def self.experience_questions
-    QUESTIONS.keys.select{|k| k.to_s.include?("_exp")}
+    Question.where(is_experience: true)
   end
 
   def self.identity_questions
-    QUESTIONS.keys.select{|k| k.to_s.include?("_given")}
-  end
-
-  def self.freeform_questions
-    [:pronouns_feel, :affinity, :notes]
-  end
-
-  def initialize(attrs={})
-    self.key = attrs[:key]
-    self.label = attrs[:label]
-  end
-
-  def context
-    "#{self.key}".gsub("_given","").gsub("klass","class").gsub("_exp", "").gsub("_","-")
-  end
-
-  def codes_field
-    "#{self.key}_codes".gsub("given","id")
-  end
-
-  def identity_field?
-    self.key.include? "_given"
+    Question.where(is_identity: true)
   end
 
-  def experience_field?
-    return true if self.key.include? "_exp"
-    return true if self.key.include? "_feel"
-    return true if self.key == "affinity"
-    return true if self.key == "notes"
+  def self.reflection_questions
+    Question.where(is_reflection: true)
   end
 
 end

app/models/response.rb

@@ -0,0 +1,19 @@
+class Response < ApplicationRecord
+
+  after_update :enqueue_export_to_graph
+
+  belongs_to :survey_response
+  belongs_to :question
+
+  def codes
+    Code.where(name: self.raw_codes)
+  end
+
+  private
+
+    # Invokes a service to update the graph databases from the associated SurveyResponse object.
+    def enqueue_export_to_graph
+      SurveyResponse.find(self.survey_response_id).save
+    end
+
+end