From 28637b5ec18ee77464a1d39cb092cb39f223929d Mon Sep 17 00:00:00 2001 From: jdarwood007 Date: Sun, 22 Oct 2023 18:43:51 -0700 Subject: [PATCH 1/2] Add support for Sphinx 3.6 Drop data and log path in favor of a base path which now contains all the files and logs. The old logic should appear if a older sphinx version is detected --- Admin-Sphinx.english.php | 3 ++ SMF 2.1/SearchAPI-Sphinxql.php | 84 +++++++++++++++++++++++----------- 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/Admin-Sphinx.english.php b/Admin-Sphinx.english.php index 24412909..60c7eede 100644 --- a/Admin-Sphinx.english.php +++ b/Admin-Sphinx.english.php @@ -15,6 +15,9 @@ $txt['sphinx_smf_tittle'] = 'SMF Settings'; $txt['sphinx_config_hints_title'] = 'Sphinx Server Configuration Hints'; +$txt['sphinx_base_path'] = 'Base data path'; +$txt['sphinx_base_path_subtext'] = 'This is the path that will be containing the search index files used by Sphinx.'; + $txt['sphinx_data_path'] = 'Index data path'; $txt['sphinx_data_path_subtext'] = 'This is the path that will be containing the search index files used by Sphinx.'; diff --git a/SMF 2.1/SearchAPI-Sphinxql.php b/SMF 2.1/SearchAPI-Sphinxql.php index 7a5948dd..7b429a4e 100644 --- a/SMF 2.1/SearchAPI-Sphinxql.php +++ b/SMF 2.1/SearchAPI-Sphinxql.php @@ -132,12 +132,9 @@ public static function searchSettings(&$config_vars) array('title', 'sphinx_server_config_tittle'), '' . $txt['sphinx_server_config_note'] . '', array('text', 'sphinx_index_name', 65, 'default_value' => 'smf', 'subtext' => $txt['sphinx_index_name_subtext']), - array('text', 'sphinx_data_path', 65, 'default_value' => '/var/sphinx/data', 'subtext' => $txt['sphinx_data_path_subtext']), - array('text', 'sphinx_log_path', 65, 'default_value' => '/var/sphinx/log', 'subtext' => $txt['sphinx_log_path_subtext']), + array('text', 'sphinx_base_path', 65, 'default_value' => '/var/sphinx/data', 'subtext' => $txt['sphinx_base_path_subtext']), array('text', 'sphinx_conf_path', 65, 'default_value' => '/etc/sphinxsearch', 'subtext' => $txt['sphinx_conf_path_subtext']), array('text', 'sphinx_bin_path', 65, 'default_value' => '/usr/bin', 'subtext' => $txt['sphinx_bin_path_subtext']), - array('text', 'sphinx_stopword_path', 65, 'default_value' => '', 'subtext' => $txt['sphinx_stopword_path_subtext']), - array('int', 'sphinx_indexer_mem', 6, 'default_value' => '32', 'subtext' => $txt['sphinx_indexer_mem_subtext'], 'postinput' => $txt['sphinx_indexer_mem_postinput']), array('int', 'sphinx_indexer_mem', 6, 'default_value' => '32', 'subtext' => $txt['sphinx_indexer_mem_subtext'], 'postinput' => $txt['sphinx_indexer_mem_postinput']), // SMF Configuration Settings. @@ -153,9 +150,6 @@ public static function searchSettings(&$config_vars) array('callback', 'SMFAction_Sphinx_Hints'), ); - // Merge them in. - $config_vars = array_merge($config_vars, $local_config_vars); - $context['post_url'] = $scripturl . '?action=admin;area=modsettings;save;sa=sphinx'; $context['settings_title'] = $txt['sphinx_server_config_tittle']; $context['sphinx_version'] = self::sphinxversion(); @@ -168,6 +162,28 @@ public static function searchSettings(&$config_vars) else $context['sphinx_version'] = '3.0'; + // Change settings for older sphinxs + if (!empty($context['sphinx_version']) && version_compare($context['sphinx_version'], '3.5', '>')) + { + $part1 = array_slice($local_config_vars, 0, 3); + $part2 = array_slice($local_config_vars, 4, 5); + $part3 = array_slice($local_config_vars, 7); + + $local_config_vars = array_merge( + $part1, + array( + array('text', 'sphinx_data_path', 65, 'default_value' => '/var/sphinx/data', 'subtext' => $txt['sphinx_data_path_subtext']), + array('text', 'sphinx_log_path', 65, 'default_value' => '/var/sphinx/log', 'subtext' => $txt['sphinx_log_path_subtext']), + array('text', 'sphinx_stopword_path', 65, 'default_value' => '', 'subtext' => $txt['sphinx_stopword_path_subtext']), + ), + $part2, + $part3 + ); + } + + // Merge them in. + $config_vars = array_merge($config_vars, $local_config_vars); + // Saving? if (isset($_GET['save'])) { @@ -943,8 +959,11 @@ function template_callback_SMFAction_Sphinx_Hints() $message = ' ' . sprintf($txt['sphinx_config_hints_desc'], $modSettings['sphinx_data_path']) . '[pre]mkdir -p ' . $modSettings['sphinx_data_path'] . ' +chmod a+w ' . $modSettings['sphinx_data_path']; + + if (!empty($context['sphinx_version']) && version_compare($context['sphinx_version'], '3.5', '>')) + $message .= ' mkdir -p ' . $modSettings['sphinx_log_path'] . ' -chmod a+w ' . $modSettings['sphinx_data_path'] . ' chmod a+w ' . $modSettings['sphinx_log_path'] . '[/pre]'; // Add a extra step for postgresql. @@ -1035,6 +1054,8 @@ function generateSphinxConfig() $host = $modSettings['sphinx_searchd_server'] == 'localhost' ? '127.0.0.1' : $modSettings['sphinx_searchd_server']; $index_name = !empty($modSettings['sphinx_index_name']) ? $modSettings['sphinx_index_name'] : 'smf'; + $modSettings['sphinx_base_path'] ??= $modSettings['sphinx_data_path']; + // Lets fall out of SMF templating and start the headers to serve a file. ob_end_clean(); ob_start(); @@ -1074,22 +1095,22 @@ function generateSphinxConfig() sql_pass = ', $db_passwd, ' sql_db = ', $db_name, ' sql_port = 3306', empty($db_character_set) ? '' : ' - sql_query_pre = SET NAMES ' . $db_character_set; + sql_query_pre = SET NAMES ' . $db_character_set; // Thanks to TheStupidOne for pgsql queries. if ($db_type == 'pgsql') echo ' - sql_query_pre = \ + sql_query_pre = \ SELECT update_settings(\'sphinx_indexed_msg_until\', (SELECT MAX(id_msg) FROM PREFIX_messages))'; else echo ' - sql_query_pre = \ + sql_query_pre = \ REPLACE INTO ', $db_prefix, 'settings (variable, value) \ SELECT \'sphinx_indexed_msg_until\', MAX(id_msg) \ FROM ', $db_prefix, 'messages'; echo ' - sql_query_range = \ + sql_query_range = \ SELECT 1, value \ FROM ', $db_prefix, 'settings \ WHERE variable = \'sphinx_indexed_msg_until\' @@ -1098,7 +1119,7 @@ function generateSphinxConfig() // Thanks to TheStupidOne for pgsql queries. if ($db_type == 'pgsql') echo ' - sql_query = \ + sql_query = \ SELECT \ m.id_msg, m.id_topic, m.id_board, CASE WHEN m.id_member = 0 THEN 4294967295 ELSE m.id_member END AS id_member, m.poster_time, m.body, m.subject, \ t.num_replies + 1 AS num_replies, CEILING(1000000 * ( \ @@ -1113,7 +1134,7 @@ function generateSphinxConfig() AND m.id_msg BETWEEN $start AND $end'; else echo ' - sql_query = \ + sql_query = \ SELECT \ m.id_msg, m.id_topic, m.id_board, IF(m.id_member = 0, 4294967295, m.id_member) AS id_member, m.poster_time, m.body, m.subject, \ t.num_replies + 1 AS num_replies, CEILING(1000000 * ( \ @@ -1127,7 +1148,9 @@ function generateSphinxConfig() AND s.variable = \'maxMsgID\' \ AND m.id_msg BETWEEN $start AND $end'; - echo ' + // Sphinx 3.5 and above this is moved. + if (!empty($context['sphinx_version']) && version_compare($context['sphinx_version'], '3.5', '>')) + echo ' sql_attr_uint = id_topic sql_attr_uint = id_board sql_attr_uint = id_member'; @@ -1138,7 +1161,8 @@ function generateSphinxConfig() sql_attr_timestamp = poster_time sql_attr_timestamp = relevance sql_attr_timestamp = num_replies'; - else + // Sphinx 3.5 moved these and changed to attr_uint. + else if (!empty($context['sphinx_version']) && version_compare($context['sphinx_version'], '3.5', '>')) echo ' sql_attr_uint = poster_time sql_attr_uint = relevance @@ -1161,24 +1185,29 @@ function generateSphinxConfig() { html_strip = 1 source = ' . $index_name . '_source - path = ', $modSettings['sphinx_data_path'], '/' . $index_name . '_sphinx_base.index', empty($modSettings['sphinx_stopword_path']) ? '' : ' - stopwords = ' . $modSettings['sphinx_stopword_path'], ' - min_word_len = 2 + stopwords = ' . ($modSettings['sphinx_stopword_path'] ?? '') . ' + min_word_len = 2'; + + // Sphinc 3.5 and above this should be located here. + if (empty($context['sphinx_version']) || (!empty($context['sphinx_version']) && version_compare($context['sphinx_version'], '3.5', '<='))) + echo ' + field = subject, body + attr_uint = id_msg, id_topic, id_board, id_member, poster_time, relevance, num_replies'; + + echo ' charset_table = 0..9, A..Z->a..z, _, a..z } index ' . $index_name . '_delta_index : ' . $index_name . '_base_index { source = ' . $index_name . '_delta_source - path = ', $modSettings['sphinx_data_path'], '/' . $index_name . '_sphinx_delta.index } index ' . $index_name . '_index { type = distributed local = ' . $index_name . '_base_index - local = ' . $index_name . '_delta_index -} + local = ' . $index_name . '_delta_index} indexer { @@ -1188,12 +1217,15 @@ function generateSphinxConfig() searchd { listen = ', !empty($modSettings['sphinx_searchd_bind']) ? $host : '0.0.0.0', ':', (empty($modSettings['sphinxql_searchd_port']) ? 9306 : (int) $modSettings['sphinxql_searchd_port']), ':mysql41 - log = ', $modSettings['sphinx_log_path'], '/searchd.log - query_log = ', $modSettings['sphinx_log_path'], '/query.log + log = + query_log = read_timeout = 5 max_children = 30 - pid_file = ', $modSettings['sphinx_data_path'], '/searchd.pid - binlog_path = ', $modSettings['sphinx_data_path'], ' +} + +common +{ + datadir = ', $modSettings['sphinx_base_path'], ' }'; die; From 85dcb43dbd17714e7908e37e905bf8ef972936ee Mon Sep 17 00:00:00 2001 From: jdarwood007 Date: Sun, 22 Oct 2023 19:29:00 -0700 Subject: [PATCH 2/2] Don't need this --- SMF 2.1/SearchAPI-Sphinxql.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SMF 2.1/SearchAPI-Sphinxql.php b/SMF 2.1/SearchAPI-Sphinxql.php index 7b429a4e..88360c5f 100644 --- a/SMF 2.1/SearchAPI-Sphinxql.php +++ b/SMF 2.1/SearchAPI-Sphinxql.php @@ -1192,7 +1192,7 @@ function generateSphinxConfig() if (empty($context['sphinx_version']) || (!empty($context['sphinx_version']) && version_compare($context['sphinx_version'], '3.5', '<='))) echo ' field = subject, body - attr_uint = id_msg, id_topic, id_board, id_member, poster_time, relevance, num_replies'; + attr_uint = id_topic, id_board, id_member, poster_time, relevance, num_replies'; echo ' charset_table = 0..9, A..Z->a..z, _, a..z