Merge pull request #13 from cre-ne-jp/keywords-rails6

キーワード抽出

app/assets/stylesheets/_custom.scss

@@ -162,10 +162,25 @@ table.day-summary {
   th {
     padding-right: 1em;
     text-align: right;
+    vertical-align: top;
     white-space: nowrap;
   }
 }
 
+a.keyword-list-time {
+  margin-left: 0.25em;
+  margin-right: 0.25em;
+  font-size: $font-size-small;
+
+  &:link, &:visited {
+    color: $gray;
+  }
+
+  &:hover {
+    color: $link-hover-color;
+  }
+}
+
 .message-visibility {
   th {
     vertical-align: top;
@@ -314,6 +329,10 @@ textarea.text-on-homepage {
   font-size: $font-size-small;
 }
 
+.keyword-num-of-privmsgs {
+  text-align: right;
+}
+
 footer {
   margin-top: 2em;
   padding: 2em 0 4em;

app/controllers/application_controller.rb

@@ -12,4 +12,15 @@ def not_authenticated
     flash[:warning] = t('views.flash.login_first')
     redirect_to(login_path)
   end
+
+  # PRIVMSG-キーワードの関連をConversationMessageの集合から抽出する
+  # @param [Array<ConversationMessage>] messages ConversationMessageの配列
+  # @return [Array<PrivmsgKeywordRelationship>]
+  def privmsg_keyword_relationships_from(messages)
+    privmsgs = messages.to_a.select { |m| m.kind_of?(Privmsg) }
+    PrivmsgKeywordRelationship
+      .includes(:keyword)
+      .from_privmsgs(privmsgs)
+      .to_a
+  end
 end

app/controllers/channels/days_controller.rb

@@ -80,6 +80,13 @@ def show
       order(:timestamp, :id).
       to_a
 
+    @privmsg_keyword_relationships =
+      privmsg_keyword_relationships_from(@conversation_messages)
+    @keywords_privmsgs_for_header = @privmsg_keyword_relationships
+      .sort_by { |r| r.privmsg.timestamp }
+      .group_by(&:keyword)
+      .map { |keyword, relations| [keyword, relations.map(&:privmsg)] }
+
     @browse_day_normal = ChannelBrowse::Day.new(
       channel: @channel, date: @date, style: :normal
     )

app/controllers/channels_controller.rb

@@ -33,6 +33,9 @@ def show
       limit(3).
       reverse
 
+    @privmsg_keyword_relationships =
+      privmsg_keyword_relationships_from(@latest_speeches)
+
     @years = MessageDate.
       where(channel: @channel).
       group(:year).

app/controllers/keywords_controller.rb

@@ -0,0 +1,37 @@
+class KeywordsController < ApplicationController
+  def index
+    relationships = PrivmsgKeywordRelationship.arel_table
+    keywords = Keyword.arel_table
+
+    privmsg_counts = relationships
+      .group(:keyword_id)
+      .project(:keyword_id, Arel.sql('COUNT(*) AS privmsg_count'))
+      .as('privmsg_counts')
+    join_cond = keywords
+      .join(privmsg_counts, Arel::Nodes::InnerJoin)
+      .on(keywords[:id].eq(privmsg_counts[:keyword_id]))
+      .join_sources
+
+    page_i = params[:page].to_i
+    page = page_i >= 1 ? page_i : 1
+    @keyword_privmsg_counts = Keyword
+      .joins(join_cond)
+      .select('keywords.*', 'privmsg_counts.privmsg_count')
+      .order('privmsg_count DESC', 'id DESC')
+      .page(page)
+  end
+
+  def show
+    @keyword = Keyword.friendly.find(params[:id])
+
+    page_i = params[:page].to_i
+    page = page_i >= 1 ? page_i : 1
+    @logs = @keyword.privmsgs
+      .page(page)
+      .joins(:channel)
+      .select(:channel_id, 'DATE(timestamp) AS date', 'channels.row_order')
+      .distinct
+      .order('date DESC', 'channels.row_order ASC')
+      .preload(:channel)
+  end
+end

app/controllers/messages/searches_controller.rb

@@ -39,8 +39,10 @@ def show
 
     if @message_search.valid?
       @result = @message_search.result
-
       @messages = @result.messages
+      @privmsg_keyword_relationships =
+        privmsg_keyword_relationships_from(@messages)
+
       set_prev_link!(path_to_prev_page(@messages))
       set_next_link!(path_to_next_page(@messages))
     else

app/helpers/application_helper.rb

@@ -75,4 +75,22 @@ def last_speech_timestamp_link(channel)
   def nav_tab_class(cond)
     cond ? 'active' : ''
   end
+
+  # PRIVMSGのキーワード部分にリンクを設定する
+  # @param [Privmsg] privmsg キーワードコマンドのPRIVMSG
+  # @param [Keyword] keyword 対応するキーワード
+  # @return [String]
+  def linkify_keyword(privmsg, keyword)
+    m = privmsg.message.match(/([ 　]+)/)
+    return sanitize(privmsg.message) unless m
+
+    command = m.pre_match
+    separator = m[1]
+    keyword_title = m.post_match
+
+    pre_keyword = sanitize("#{command}#{separator}")
+    link_to_keyword = link_to(sanitize(keyword_title), keyword_path(keyword))
+
+    raw("#{pre_keyword}#{link_to_keyword}")
+  end
 end

app/models/keyword.rb

@@ -0,0 +1,66 @@
+require 'nkf'
+require 'strscan'
+
+class Keyword < ApplicationRecord
+  extend FriendlyId
+  friendly_id :title
+
+  has_many :privmsg_keyword_relationships, dependent: :destroy
+  has_many :privmsgs, through: :privmsg_keyword_relationships
+
+  validates(:title,
+            presence: true)
+  validates(:display_title,
+            presence: true)
+
+  # 全文検索
+  scope :full_text_search, ->query {
+    where('MATCH(display_title) AGAINST(? IN BOOLEAN MODE)', "*D+ #{query}")
+  }
+
+  # キーワードの表記を正規化する
+  # @param [String] text 正規化するテキスト
+  # @return [String]
+  #
+  # * 英数字といくつかの記号を半角にする
+  # * 半角片仮名を全角にする
+  # * 全角空白を半角にする
+  # * パスに使われる記号をアンダーバーに置換する
+  # * 空白をアンダーバーに置換する
+  # * アンダーバーの出現回数を最小化する
+  def self.normalize(text)
+    nkfed = NKF.nkf('-Ww -Z0 -Z1 -X -m0', text)
+
+    # URL生成時に失敗しないよう、パスに使われる記号を置換する
+    symbols_for_path_replaced = nkfed.gsub(%r![./]!, '_')
+
+    spaces_replaced = symbols_for_path_replaced.gsub(/\s/, '_')
+    underbars_trimmed = spaces_replaced.
+      sub(/\A_+/, '').
+      sub(/_+\z/, '')
+
+    underbars_shortened = String.new(underbars_trimmed)
+    ss = StringScanner.new(underbars_shortened)
+
+    # 以下の3通りのいずれか
+    #
+    # 1. 英数字以外 アンダーバー 任意の文字
+    # 2. 英数字 アンダーバー 英数字以外
+    # 3. 英数字 アンダーバー2個以上 英数字
+    re = /([^_A-Z0-9])(_+)[^_]|([A-Z0-9])(_+)[^_A-Z0-9]|([A-Z0-9]_)(_+)[A-Z0-9]/i
+    while ss.scan_until(re)
+      pre_underbars_length = (ss[1] || ss[3] || ss[5]).length
+      underbars_length = (ss[2] || ss[4] || ss[6]).length
+
+      # 余分なアンダーバーを除去する
+      underbars_shortened.slice!(ss.pre_match.length + pre_underbars_length,
+                                 underbars_length)
+
+      # 除去したアンダーバーの文字数も考慮しながら
+      # マッチした部分の最後の文字まで戻す
+      ss.pos -= (underbars_length + 1)
+    end
+
+    underbars_shortened.downcase
+  end
+end

app/models/privmsg.rb

@@ -1,4 +1,7 @@
 class Privmsg < ConversationMessage
+  has_one :privmsg_keyword_relationship, dependent: :destroy
+  has_one :keyword, through: :privmsg_keyword_relationship
+
   validates :message, length: { minimum: 1 }
 
   # Tiarra のログ形式の文字列を返す

app/models/privmsg_keyword_relationship.rb

@@ -0,0 +1,12 @@
+class PrivmsgKeywordRelationship < ApplicationRecord
+  belongs_to :privmsg
+  belongs_to :keyword
+
+  validates :privmsg_id, presence: true
+  validates :keyword_id, presence: true
+
+  scope :from_privmsgs, ->privmsgs {
+    includes(:keyword)
+      .where(privmsg: privmsgs)
+  }
+end

app/views/channels/days/show.html.erb

@@ -28,6 +28,23 @@
                     <th scope="row">発言数</th>
                     <td><%= @conversation_messages.length %></td>
                   </tr>
+                  <% unless @privmsg_keyword_relationships.empty? %>
+                    <tr id="keyword-list">
+                      <th scope="row">キーワード</th>
+                      <td>
+                        <ul class="list-unstyled">
+                          <% @keywords_privmsgs_for_header.each do |keyword, privmsgs| %>
+                            <li>
+                              <%= link_to(sanitize(keyword.display_title), keyword, class: 'channel-day-keyword-link') %>
+                              <% privmsgs.each do |m| %>
+                                <%= link_to(m.timestamp.strftime('%T'), "##{m.fragment_id}", class: 'keyword-list-time') %>
+                              <% end %>
+                            </li>
+                          <% end %>
+                        </ul>
+                      </td>
+                    </tr>
+                  <% end %>
                   <tr class="message-visibility">
                     <th scope="row">表示</th>
                     <td>
@@ -72,7 +89,7 @@
                 <%= render(partial: 'raw_message', collection: @sorted_messages, as: :m) %>
               </div>
             <% else %>
-              <%= render('shared/message_list', messages: @sorted_messages, show_channel: false, style: :normal) %>
+              <%= render('shared/message_list', messages: @sorted_messages, privmsg_keyword_relationships: @privmsg_keyword_relationships, show_channel: false, style: :normal) %>
             <% end %>
 
             <div class="alert alert-info no-message-to-show">表示するメッセージがありません。</div>

app/views/channels/show.html.erb

@@ -32,7 +32,7 @@
 
             <% unless @latest_speeches.empty? %>
               <h2>最新の発言</h2>
-              <%= render('shared/message_with_datetime_list', messages: @latest_speeches, show_channel: false) %>
+              <%= render('shared/message_with_datetime_list', messages: @latest_speeches, privmsg_keyword_relationships: @privmsg_keyword_relationships, show_channel: false) %>
             <% end %>
 
             <h2>年別のログ</h2>

app/views/keywords/index.html.erb

@@ -0,0 +1,37 @@
+<% title('キーワード') %>
+<div class="container main-container">
+  <div class="row">
+    <div class="col-xs-12">
+      <div class="main-panel">
+        <div class="panel-body">
+          <%= render('shared/flash') %>
+
+          <div class="page-header">
+            <h1>キーワード</h1>
+          </div>
+
+          <p><%= page_entries_info(@keyword_privmsg_counts) %></p>
+
+          <table class="table table-striped keyword-list">
+            <thead>
+              <tr>
+                <th class="keyword-keyword">キーワード</th>
+                <th class="keyword-num-of-privmsgs">発言数</th>
+              </tr>
+            </thead>
+            <tbody>
+              <% @keyword_privmsg_counts.each do |keyword| %>
+                <tr>
+                  <td class="keyword-keyword"><%= link_to(h(keyword.display_title), keyword) %></td>
+                  <td class="keyword-num-of-privmsgs"><%= keyword.privmsg_count %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+
+          <%= paginate(@keyword_privmsg_counts, outer_window: 2) %>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

app/views/keywords/show.html.erb

@@ -0,0 +1,48 @@
+<% title(@keyword.display_title) %>
+<div class="container main-container">
+  <div class="row">
+    <div class="col-xs-12">
+      <div class="main-panel">
+        <div class="panel-body">
+          <%= render('shared/flash') %>
+
+          <ol class="breadcrumb">
+            <li><%= link_to('キーワード', keywords_path) %></li>
+            <li class="active"><%= @keyword.display_title %></li>
+          </ol>
+
+          <article>
+            <div class="page-header">
+              <h1>キーワード「<%= @keyword.display_title %>」を含むログ</h1>
+            </div>
+
+            <p><%= page_entries_info(@logs, entry_name: 'ログ') %></p>
+
+            <table class="table table-striped channel-date-list">
+              <thead>
+                <tr>
+                  <th class="channel-date-channel">チャンネル</th>
+                  <th class="channel-date-date">日付</th>
+                </tr>
+              </thead>
+              <tbody>
+                <% style = message_list_style(cookies) %>
+                <% @logs.each do |log| %>
+                  <% channel = log.channel %>
+                  <% date = log.date %>
+                  <% browse_day = ChannelBrowse::Day.new(channel: channel, date: date, style: style) %>
+                  <tr>
+                    <td class="channel-date-channel"><%= link_to(channel.name_with_prefix, channel) %></td>
+                    <td class="channel-date-date"><%= link_to(date.strftime('%F'), browse_day.path) %></td>
+                  </tr>
+                <% end %>
+              </tbody>
+            </table>
+
+            <%= paginate(@logs, outer_window: 2) %>
+          </article>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

app/views/messages/searches/_message_group.html.erb

@@ -2,4 +2,4 @@
 
 <h2><%= date.strftime('%F') %></h2>
 
-<%= render('shared/message_list', messages: messages, show_channel: true, style: message_list_style(cookies)) %>
+<%= render('shared/message_list', messages: messages, privmsg_keyword_relationships: privmsg_keyword_relationships, show_channel: true, style: message_list_style(cookies)) %>

app/views/messages/searches/show.html.erb

@@ -34,7 +34,9 @@
 
           <p><%= page_entries_info(@messages) %></p>
 
-          <%= render(partial: 'message_group', collection: @result.message_groups.to_a) %>
+          <% privmsg_keyword_relationships ||= [] %>
+          <% privmsg_id_keyword_map = privmsg_keyword_relationships.map { |r| [r.privmsg_id, r.keyword] }.to_h %>
+          <%= render(partial: 'message_group', collection: @result.message_groups.to_a, locals: { privmsg_keyword_relationships: @privmsg_keyword_relationships, privmsg_id_keyword_map: privmsg_id_keyword_map }) %>
 
           <%= paginate(@messages, outer_window: 2) %>
         </div>