Add gitlab private token for --wapp-url

Signed-off-by: bretfourbe <[email protected]>

wapitiCore/attack/mod_wapp.py

@@ -22,7 +22,7 @@
 import string
 from typing import Dict, Tuple, Optional, List
 import re
-from urllib.parse import urlparse
+from urllib.parse import urlparse, quote_plus
 
 from httpx import RequestError
 
@@ -134,6 +134,8 @@ class ModuleWapp(Attack):
 
     BASE_URL = Attack.wapp_url
     WAPP_DIR = Attack.wapp_dir
+    # Store the gitlab private token from env variable
+    GITLAB_PRIVATE_TOKEN = os.environ.get("GITLAB_PRIVATE_TOKEN", None)
     WAPP_CATEGORIES = "categories.json"
     WAPP_GROUPS = "groups.json"
     WAPP_TECHNOLOGIES = "technologies.json"
@@ -168,9 +170,12 @@ async def copy_files_to_conf(self, files_to_copy: List[str]):
     async def update(self):
         """Update the Wappalizer database from the web and load the patterns."""
 
-        wapp_categories_url = f"{self.BASE_URL}src/categories.json"
-        wapp_technologies_base_url = f"{self.BASE_URL}src/technologies/"
-        wapp_groups_url = f"{self.BASE_URL}src/groups.json"
+        sources = ["src/categories.json", "src/technologies/", "src/groups.json"]
+        if self.GITLAB_PRIVATE_TOKEN:
+            sources = [quote_plus(src) for src in sources]
+        wapp_categories_url = f"{self.BASE_URL}{sources[0]}"
+        wapp_technologies_base_url = f"{self.BASE_URL}{sources[1]}"
+        wapp_groups_url = f"{self.BASE_URL}{sources[2]}"
         if self.WAPP_DIR:
             categories_file_path = os.path.join(self.WAPP_DIR, self.WAPP_CATEGORIES)
             groups_file_path = os.path.join(self.WAPP_DIR, self.WAPP_GROUPS)
@@ -342,9 +347,16 @@ async def _load_wapp_database(self, categories_url: str, technologies_base_url:
 
         # Requesting all technologies one by one
         for technology_file_name in technology_files_names:
-            request = Request(technologies_base_url + technology_file_name)
+            technologies_file_url = technologies_base_url + technology_file_name
+            headers={}
+            if self.GITLAB_PRIVATE_TOKEN:
+                technologies_file_url = technologies_file_url + "/raw"
+                headers = {"PRIVATE_TOKEN": self.GITLAB_PRIVATE_TOKEN}
+
+            request = Request(technologies_file_url)
+
             try:
-                response: Response = await self.crawler.async_send(request)
+                response: Response = await self.crawler.async_send(request, headers=headers)
             except RequestError:
                 self.network_errors += 1
                 raise