Merge branch 'searxng:master' into master

Dockerfile

@@ -35,7 +35,6 @@ RUN apk add --no-cache -t build-dependencies \
     git \
  && apk add --no-cache \
     ca-certificates \
-    su-exec \
     python3 \
     py3-pip \
     libxml2 \

dockerfiles/docker-entrypoint.sh

@@ -175,4 +175,4 @@ unset MORTY_KEY
 
 # Start uwsgi
 printf 'Listen on %s\n' "${BIND_ADDRESS}"
-exec su-exec searxng:searxng uwsgi --master --http-socket "${BIND_ADDRESS}" "${UWSGI_SETTINGS_PATH}"
+exec uwsgi --master --uid searxng --gid searxng --http-socket "${BIND_ADDRESS}" "${UWSGI_SETTINGS_PATH}"

docs/dev/engines/engine_overview.rst

@@ -339,6 +339,8 @@ type.
    content                   *(not implemented yet)*
    publishedDate             :py:class:`datetime.datetime`, time of publish
    thumbnail                 string, url to a small-preview image
+   length                    :py:class:`datetime.timedelta`, duration of result
+   views                     string, view count in humanized number format
    ========================= =====================================================
 
 

docs/dev/engines/online/geizhals.rst

@@ -0,0 +1,8 @@
+.. _gitea geizhals:
+
+========
+Geizhals
+========
+
+.. automodule:: searx.engines.geizhals
+  :members:

requirements-dev.txt

@@ -2,12 +2,12 @@ mock==5.1.0
 nose2[coverage_plugin]==0.15.1
 cov-core==1.15.0
 black==24.3.0
-pylint==3.2.5
+pylint==3.2.6
 splinter==0.21.0
-selenium==4.23.0
+selenium==4.23.1
 Pallets-Sphinx-Themes==2.1.3
 Sphinx<=7.1.2; python_version == '3.8'
-Sphinx==7.3.7; python_version > '3.8'
+Sphinx==7.4.7; python_version > '3.8'
 sphinx-issues==4.1.0
 sphinx-jinja==2.0.2
 sphinx-tabs==3.4.5

searx/engines/discourse.py

@@ -10,6 +10,8 @@
 - :py:obj:`api_order`
 - :py:obj:`search_endpoint`
 - :py:obj:`show_avatar`
+- :py:obj:`api_key`
+- :py:obj:`api_username`
 
 Example
 =======
@@ -27,6 +29,20 @@
      categories: ['social media', 'sports']
      show_avatar: true
 
+If the forum is private, you need to add an API key and username for the search:
+
+.. code:: yaml
+
+   - name: paddling
+     engine: discourse
+     shortcut: paddle
+     base_url: 'https://forums.paddling.com/'
+     api_order: views
+     categories: ['social media', 'sports']
+     show_avatar: true
+     api_key: '<KEY>'
+     api_username: 'system'
+
 
 Implementations
 ===============
@@ -65,6 +81,12 @@
 show_avatar = False
 """Show avatar of the user who send the post."""
 
+api_key = ''
+"""API key of the Discourse forum."""
+
+api_username = ''
+"""API username of the Discourse forum."""
+
 paging = True
 time_range_support = True
 
@@ -98,6 +120,12 @@ def request(query, params):
         'X-Requested-With': 'XMLHttpRequest',
     }
 
+    if api_key != '':
+        params['headers']['Api-Key'] = api_key
+
+    if api_username != '':
+        params['headers']['Api-Username'] = api_username
+
     return params
 
 

searx/engines/geizhals.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Geizhals is a German website to compare the price of a product on the
+most common German shopping sites and find the lowest price.
+
+The sorting of the search results can be influenced by the following additions
+to the search term:
+
+``asc`` or ``price``
+  To sort by price in ascending order.
+
+``desc``
+  To sort by price in descending order.
+
+"""
+
+import re
+
+from urllib.parse import urlencode
+from lxml import html
+
+from searx.utils import eval_xpath, eval_xpath_list, extract_text
+
+about = {
+    'website': 'https://geizhals.de',
+    'wikidata_id': 'Q15977657',
+    'use_official_api': False,
+    'official_api_documentation': None,
+    'require_api_key': False,
+    'results': 'HTML',
+    'language': 'de',
+}
+paging = True
+categories = ['shopping']
+
+base_url = "https://geizhals.de"
+sort_order = 'relevance'
+
+SORT_RE = re.compile(r"sort:(\w+)")
+sort_order_map = {
+    'relevance': None,
+    'price': 'p',
+    'asc': 'p',
+    'desc': '-p',
+}
+
+
+def request(query, params):
+    sort = None
+
+    sort_order_path = SORT_RE.search(query)
+    if sort_order_path:
+        sort = sort_order_map.get(sort_order_path.group(1))
+        query = SORT_RE.sub("", query)
+        logger.debug(query)
+
+    args = {
+        'fs': query,
+        'pg': params['pageno'],
+        'toggle_all': 1,  # load item specs
+        'sort': sort,
+    }
+    params['url'] = f"{base_url}/?{urlencode(args)}"
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+    for result in eval_xpath_list(dom, "//article[contains(@class, 'listview__item')]"):
+        content = []
+        for spec in eval_xpath_list(result, ".//div[contains(@class, 'specs-grid__item')]"):
+            content.append(f"{extract_text(eval_xpath(spec, './dt'))}: {extract_text(eval_xpath(spec, './dd'))}")
+
+        metadata = [
+            extract_text(eval_xpath(result, ".//div[contains(@class, 'stars-rating-label')]")),
+            extract_text(eval_xpath(result, ".//div[contains(@class, 'listview__offercount')]")),
+        ]
+
+        item = {
+            'template': 'products.html',
+            'url': (
+                base_url + "/" + extract_text(eval_xpath(result, ".//a[contains(@class, 'listview__name-link')]/@href"))
+            ),
+            'title': extract_text(eval_xpath(result, ".//h3[contains(@class, 'listview__name')]")),
+            'content': ' | '.join(content),
+            'thumbnail': extract_text(eval_xpath(result, ".//img[contains(@class, 'listview__image')]/@src")),
+            'price': "Bestes Angebot: "
+            + extract_text(eval_xpath(result, ".//a[contains(@class, 'listview__price-link')]")).split(" ")[1]
+            + "€",
+            'metadata': ', '.join(item for item in metadata if item),
+        }
+
+        results.append(item)
+
+    return results

searx/engines/invidious.py

@@ -7,6 +7,8 @@
 from urllib.parse import quote_plus, urlparse
 from dateutil import parser
 
+from searx.utils import humanize_number
+
 # about
 about = {
     "website": 'https://api.invidious.io/',
@@ -91,7 +93,8 @@ def response(resp):
                     "url": url,
                     "title": result.get("title", ""),
                     "content": result.get("description", ""),
-                    'length': length,
+                    "length": length,
+                    "views": humanize_number(result['viewCount']),
                     "template": "videos.html",
                     "author": result.get("author"),
                     "publishedDate": publishedDate,

searx/engines/peertube.py

@@ -14,7 +14,7 @@
 
 from searx.network import get  # see https://github.com/searxng/searxng/issues/762
 from searx.locales import language_tag
-from searx.utils import html_to_text
+from searx.utils import html_to_text, humanize_number
 from searx.enginelib.traits import EngineTraits
 
 traits: EngineTraits
@@ -124,6 +124,7 @@ def video_response(resp):
                 'content': html_to_text(result.get('description') or ''),
                 'author': result.get('account', {}).get('displayName'),
                 'length': minute_to_hm(result.get('duration')),
+                'views': humanize_number(result['views']),
                 'template': 'videos.html',
                 'publishedDate': parse(result['publishedAt']),
                 'iframe_src': result.get('embedUrl'),

searx/engines/piped.py

@@ -53,6 +53,8 @@
 import datetime
 from dateutil import parser
 
+from searx.utils import humanize_number
+
 # about
 about = {
     "website": 'https://github.com/TeamPiped/Piped/',
@@ -138,6 +140,7 @@ def response(resp):
             "title": result.get("title", ""),
             "publishedDate": parser.parse(time.ctime(uploaded / 1000)) if uploaded != -1 else None,
             "iframe_src": _frontend_url() + '/embed' + result.get("url", ""),
+            "views": humanize_number(result["views"]),
         }
         length = result.get("duration")
         if length:

searx/settings.yml

@@ -797,6 +797,11 @@ engines:
     timeout: 8.0
     disabled: true
 
+  - name: geizhals
+    engine: geizhals
+    shortcut: geiz
+    disabled: true
+
   - name: genius
     engine: genius
     shortcut: gen
@@ -2157,10 +2162,13 @@ engines:
 
   - name: yacy images
     engine: yacy
+    network: yacy
     categories: images
     search_type: image
     shortcut: yai
     disabled: true
+    # timeout can be reduced in 'local' search mode
+    timeout: 5.0
 
   - name: rumble
     engine: rumble

searx/static/themes/simple/src/less/style.less

@@ -271,6 +271,7 @@ article[data-vim-selected].category-social {
 
   .published_date,
   .result_length,
+  .result_views,
   .result_author,
   .result_shipping,
   .result_source_country {

searx/templates/simple/macros.html

@@ -33,6 +33,7 @@ <h3>{{ result_link(result.url, result.title|safe) }}</h3>
 {%- macro result_sub_header(result) -%}
   {%- if result.publishedDate %}<time class="published_date" datetime="{{ result.pubdate }}" >{{ result.publishedDate }}</time>{% endif -%}
   {%- if result.length %}<div class="result_length">{{ _('Length') }}: {{ result.length }}</div>{% endif -%}
+  {%- if result.views %}<div class="result_views">{{ _('Views') }}: {{ result.views }}</div>{% endif -%}
   {%- if result.author %}<div class="result_author">{{ _('Author') }}: {{ result.author }}</div>{% endif -%}
   {%- if result.metadata %}<div class="highlight">{{ result.metadata|safe }}</div>{% endif -%}
 {%- endmacro -%}

searx/templates/simple/result_templates/products.html

@@ -7,7 +7,7 @@
 {% if result.source_country %}<div class="result_source_country">{{ result.source_country }}</div></br>{% endif %}
 {%- if result.content %}
   <p class="content">
-    {{ result.content }}
+    {{ result.content|safe }}
   </p>
 {% endif -%}
 {{- result_sub_footer(result, proxify) -}}

searx/utils.py

@@ -44,16 +44,6 @@
 _JS_VOID_RE = re.compile(r'void\s+[0-9]+|void\s*\([0-9]+\)')
 _JS_DECIMAL_RE = re.compile(r":\s*\.")
 
-_STORAGE_UNIT_VALUE: Dict[str, int] = {
-    'TB': 1024 * 1024 * 1024 * 1024,
-    'GB': 1024 * 1024 * 1024,
-    'MB': 1024 * 1024,
-    'TiB': 1000 * 1000 * 1000 * 1000,
-    'GiB': 1000 * 1000 * 1000,
-    'MiB': 1000 * 1000,
-    'KiB': 1000,
-}
-
 _XPATH_CACHE: Dict[str, XPath] = {}
 _LANG_TO_LC_CACHE: Dict[str, Dict[str, str]] = {}
 
@@ -344,6 +334,18 @@ def humanize_bytes(size, precision=2):
     return "%.*f %s" % (precision, size, s[p])
 
 
+def humanize_number(size, precision=0):
+    """Determine the *human readable* value of a decimal number."""
+    s = ['', 'K', 'M', 'B', 'T']
+
+    x = len(s)
+    p = 0
+    while size > 1000 and p < x:
+        p += 1
+        size = size / 1000.0
+    return "%.*f%s" % (precision, size, s[p])
+
+
 def convert_str_to_int(number_str: str) -> int:
     """Convert number_str to int or 0 if number_str is not a number."""
     if number_str.isdigit():