Merge branch 'master' into filter

CONTRIBUTORS.md

@@ -83,6 +83,7 @@
 - [jxdv](https://github.com/jxdv)
 - [Xeonacid](https://github.com/Xeonacid)
 - [Valentijn Scholten](https://www.github.com/valentijnscholten)
+- [partoneplay](https://github.com/partoneplay)
 
 
 Special thanks to all the people who are named here!

config.ini

@@ -14,6 +14,7 @@ exclude-subdirs = %%ff/,.;/,..;/,;/,./,../,%%2e/,%%2e%%2e/
 random-user-agents = False
 max-time = 0
 exit-on-error = False
+skip-on-status = 429
 #filter-threshold = 10
 #subdirs = /,api/
 #include-status = 200-299,401
@@ -26,7 +27,6 @@ exit-on-error = False
 #exclude-regex = "^403$"
 #exclude-redirect = "*/error.html"
 #exclude-response = 404.html
-#skip-on-status = 429,999
 
 [dictionary]
 default-extensions = php,asp,aspx,jsp,html,htm

db/dicc.txt

@@ -3047,21 +3047,32 @@ api/apidocs/swagger.json
 api/application.wadl
 api/batch
 api/cask/graphql
+api/chat
 api/config
 api/config.json
+api/copy
+api/create
 api/credential.json
 api/credentials.json
 api/database.json
+api/delete
 api/docs
 api/docs/
+api/embed
+api/embeddings
 api/error_log
+api/generate
 api/index.html
 api/jsonws
 api/jsonws/invoke
 api/login.json
 api/package_search/v4/documentation
 api/profile
 api/proxy
+api/ps
+api/pull
+api/push
+api/show
 api/snapshots
 api/spec/swagger.json
 api/swagger
@@ -3073,6 +3084,7 @@ api/swagger/index.html
 api/swagger/static/index.html
 api/swagger/swagger
 api/swagger/ui/index
+api/tags
 api/timelion/run
 api/user.json
 api/users.json
@@ -8928,9 +8940,19 @@ v1.0/
 v1.1
 v1/
 v1/api-docs
+v1/audio/speech
+v1/batches
+v1/chat/completions
+v1/embeddings
+v1/files
+v1/fine_tuning/jobs
+v1/images/generations
+v1/models
+v1/moderations
 v1/public/yql
 v1/test/js/console.html
 v1/test/js/console_ajax.js
+v1/uploads
 v2
 v2.0
 v2/

dirsearch.py

@@ -66,6 +66,11 @@ def main():
 
     options.update(parse_options())
 
+    if options["session_file"]:
+        print("WARNING: Running an untrusted session file might lead to unwanted code execution!")
+        if input("[c]ontinue / [q]uit: ") != "c":
+            exit(1)
+
     from lib.controller.controller import Controller
 
     Controller()

lib/connection/requester.py

@@ -101,21 +101,6 @@ def set_url(self, url: str) -> None:
     def set_header(self, key: str, value: str) -> None:
         self.headers[key] = value.lstrip()
 
-    def set_proxy(self, proxy: str) -> None:
-        if not proxy:
-            return
-
-        if not proxy.startswith(PROXY_SCHEMES):
-            proxy = f"http://{proxy}"
-
-        if self.proxy_cred and "@" not in proxy:
-            # socks5://localhost:9050 => socks5://[credential]@localhost:9050
-            proxy = proxy.replace("://", f"://{self.proxy_cred}@", 1)
-
-        self.session.proxies = {"https": proxy}
-        if not proxy.startswith("https://"):
-            self.session.proxies["http"] = proxy
-
     def is_rate_exceeded(self) -> bool:
         return self._rate >= options["max_rate"] > 0
 
@@ -188,16 +173,24 @@ def request(self, path: str, proxy: str | None = None) -> Response:
         self.increase_rate()
 
         err_msg = None
-
-        # Safe quote all special characters to prevent them from being encoded
-        url = safequote(self._url + path if self._url else path)
+        url = self._url + safequote(path)
 
         # Why using a loop instead of max_retries argument? Check issue #1009
         for _ in range(options["max_retries"] + 1):
             try:
+                proxies = {}
                 try:
-                    proxy = proxy or random.choice(options["proxies"])
-                    self.set_proxy(proxy)
+                    proxy_url = proxy or random.choice(options["proxies"])
+                    if not proxy_url.startswith(PROXY_SCHEMES):
+                        proxy_url = f"http://{proxy_url}"
+
+                    if self.proxy_cred and "@" not in proxy_url:
+                        # socks5://localhost:9050 => socks5://[credential]@localhost:9050
+                        proxy_url = proxy_url.replace("://", f"://{self.proxy_cred}@", 1)
+
+                    proxies["https"] = proxy_url
+                    if not proxy_url.startswith("https://"):
+                        proxies["http"] = proxy_url
                 except IndexError:
                     pass
 
@@ -212,16 +205,17 @@ def request(self, path: str, proxy: str | None = None) -> Response:
                     headers=self.headers,
                     data=options["data"],
                 )
-                prepped = self.session.prepare_request(request)
-                prepped.url = url
+                prep = self.session.prepare_request(request)
+                prep.url = url
 
                 origin_response = self.session.send(
-                    prepped,
+                    prep,
                     allow_redirects=options["follow_redirects"],
                     timeout=options["timeout"],
+                    proxies=proxies,
                     stream=True,
                 )
-                response = Response(origin_response)
+                response = Response(url, origin_response)
 
                 log_msg = f'"{options["http_method"]} {response.url}" {response.status} - {response.length}B'
 
@@ -359,24 +353,21 @@ async def replay_request(self, path: str, proxy: str) -> AsyncResponse:
                 mounts={"all://": transport},
                 timeout=httpx.Timeout(options["timeout"]),
             )
-        return await self.request(path, self.replay_session)
+        return await self.request(path, self.replay_session, replay=True)
 
     # :path: is expected not to start with "/"
     async def request(
-        self, path: str, session: httpx.AsyncClient | None = None
+        self, path: str, session: httpx.AsyncClient | None = None, replay: bool = False
     ) -> AsyncResponse:
         while self.is_rate_exceeded():
             await asyncio.sleep(0.1)
 
         self.increase_rate()
 
         err_msg = None
-
-        # Safe quote all special characters to prevent them from being encoded
-        url = safequote(self._url + path if self._url else path)
-        parsed_url = urlparse(url)
-
+        url = self._url + safequote(path)
         session = session or self.session
+
         for _ in range(options["max_retries"] + 1):
             try:
                 if self.agents:
@@ -388,16 +379,15 @@ async def request(
                     url,
                     headers=self.headers,
                     data=options["data"],
+                    extensions={"target": (url if replay else f"/{safequote(path)}").encode()},
                 )
-                if p := parsed_url.path:
-                    request.extensions = {"target": p.encode()}
 
                 xresponse = await session.send(
                     request,
                     stream=True,
                     follow_redirects=options["follow_redirects"],
                 )
-                response = await AsyncResponse.create(xresponse)
+                response = await AsyncResponse.create(url, xresponse)
                 await xresponse.aclose()
 
                 log_msg = f'"{options["http_method"]} {response.url}" {response.status} - {response.length}B'

lib/connection/response.py

@@ -35,9 +35,9 @@
 
 
 class BaseResponse:
-    def __init__(self, response: requests.Response | httpx.Response) -> None:
+    def __init__(self, url, response: requests.Response | httpx.Response) -> None:
         self.datetime = time.strftime("%Y-%m-%d %H:%M:%S")
-        self.url = str(response.url)
+        self.url = url
         self.full_path = parse_path(self.url)
         self.path = clean_path(self.full_path)
         self.status = response.status_code
@@ -77,8 +77,8 @@ def __eq__(self, other: Any) -> bool:
 
 
 class Response(BaseResponse):
-    def __init__(self, response: requests.Response) -> None:
-        super().__init__(response)
+    def __init__(self, url, response: requests.Response) -> None:
+        super().__init__(url, response)
 
         for chunk in response.iter_content(chunk_size=ITER_CHUNK_SIZE):
             self.body += chunk
@@ -99,8 +99,8 @@ def __init__(self, response: requests.Response) -> None:
 
 class AsyncResponse(BaseResponse):
     @classmethod
-    async def create(cls, response: httpx.Response) -> AsyncResponse:
-        self = cls(response)
+    async def create(cls, url, response: httpx.Response) -> AsyncResponse:
+        self = cls(url, response)
         async for chunk in response.aiter_bytes(chunk_size=ITER_CHUNK_SIZE):
             self.body += chunk
 

lib/controller/controller.py

@@ -72,11 +72,6 @@
 class Controller:
     def __init__(self) -> None:
         if options["session_file"]:
-            print("WARNING: Running an untrusted session file might lead to unwanted code execution!")
-            interface.in_line("[c]continue / [q]uit: ")
-            if input() != "c":
-                exit(1)
-
             self._import(options["session_file"])
             self.old_session = True
         else:
@@ -205,12 +200,9 @@ def run(self) -> None:
         self.requester = Requester()
         if options["async_mode"]:
             self.loop = asyncio.new_event_loop()
-            try:
-                self.loop.add_signal_handler(signal.SIGINT, self.handle_pause)
-            except NotImplementedError:
-                # Windows
-                signal.signal(signal.SIGINT, self.handle_pause)
-                signal.signal(signal.SIGTERM, self.handle_pause)
+
+        signal.signal(signal.SIGINT, lambda *_: self.handle_pause())
+        signal.signal(signal.SIGTERM, lambda *_: self.handle_pause())
 
         while options["urls"]:
             url = options["urls"][0]
@@ -514,18 +506,14 @@ def is_timed_out(self) -> bool:
 
     def process(self) -> None:
         while True:
-            try:
-                while not self.fuzzer.is_finished():
-                    if self.is_timed_out():
-                        raise SkipTargetInterrupt(
-                            "Runtime exceeded the maximum set by the user"
-                        )
-                    time.sleep(0.5)
-
-                break
+            while not self.fuzzer.is_finished():
+                if self.is_timed_out():
+                    raise SkipTargetInterrupt(
+                        "Runtime exceeded the maximum set by the user"
+                    )
+                time.sleep(0.5)
 
-            except KeyboardInterrupt:
-                self.handle_pause()
+            break
 
     def add_directory(self, path: str) -> None:
         """Add directory to the recursion queue"""

lib/core/fuzzer.py

@@ -154,6 +154,7 @@ def __init__(
             not_found_callbacks=not_found_callbacks,
             error_callbacks=error_callbacks,
         )
+        self._exc: Exception | None = None
         self._threads = []
         self._play_event = threading.Event()
         self._quit_event = threading.Event()
@@ -208,13 +209,14 @@ def start(self) -> None:
         self.setup_scanners()
         self.setup_threads()
         self.play()
+        self._quit_event.clear()
 
         for thread in self._threads:
             thread.start()
 
     def is_finished(self) -> bool:
-        if self.exc:
-            raise self.exc
+        if self._exc:
+            raise self._exc
 
         for thread in self._threads:
             if thread.is_alive():
@@ -238,7 +240,12 @@ def quit(self) -> None:
 
     def scan(self, path: str) -> None:
         scanners = self.get_scanners_for(path)
-        response = self._requester.request(path)
+        try:
+            response = self._requester.request(path)
+        except RequestException as e:
+            for callback in self.error_callbacks:
+                callback(e)
+            return
 
         if self.is_excluded(response):
             for callback in self.not_found_callbacks:
@@ -274,11 +281,8 @@ def thread_proc(self) -> None:
             except StopIteration:
                 break
 
-            except RequestException as e:
-                for callback in self.error_callbacks:
-                    callback(e)
-
-                continue
+            except Exception as e:
+                self._exc = e
 
             finally:
                 time.sleep(options["delay"])
@@ -370,12 +374,6 @@ async def start(self) -> None:
 
         await asyncio.gather(*self._background_tasks)
 
-    def is_finished(self) -> bool:
-        if self.exc:
-            raise self.exc
-
-        return len(self._background_tasks) == 0
-
     def play(self) -> None:
         self._play_event.set()
 
@@ -388,7 +386,12 @@ def quit(self) -> None:
 
     async def scan(self, path: str) -> None:
         scanners = self.get_scanners_for(path)
-        response = await self._requester.request(path)
+        try:
+            response = await self._requester.request(path)
+        except RequestException as e:
+            for callback in self.error_callbacks:
+                callback(e)
+            return
 
         if self.is_excluded(response):
             for callback in self.not_found_callbacks:
@@ -422,8 +425,5 @@ async def task_proc(self) -> None:
                 await self.scan(self._base_path + path)
             except StopIteration:
                 pass
-            except RequestException as e:
-                for callback in self.error_callbacks:
-                    callback(e)
             finally:
                 await asyncio.sleep(options["delay"])