feat: support arkose in chat

backend/api/conf/config.py

@@ -74,6 +74,8 @@ class OpenaiWebChatGPTSetting(BaseModel):
     chatgpt_base_url: Optional[str] = None
     proxy: Optional[str] = None
     wss_proxy: Optional[str] = None
+    enable_arkose_endpoint: bool = False
+    arkose_endpoint_base: Optional[str] = None
     common_timeout: int = Field(20, ge=1,
                                 description="Increase this value if timeout error occurs.")  # connect, read, write
     ask_timeout: int = Field(600, ge=1)
@@ -93,6 +95,13 @@ def chatgpt_base_url_end_with_slash(cls, v):
             v += '/'
         return v
 
+    @field_validator("arkose_endpoint_base")
+    @classmethod
+    def arkose_endpoint_base_end_with_slash(cls, v):
+        if v is not None and not v.endswith('/'):
+            v += '/'
+        return v
+
     @field_validator("model_code_mapping")
     @classmethod
     def check_all_model_key_appears(cls, v):

backend/api/exceptions.py

@@ -5,7 +5,7 @@ class SelfDefinedException(Exception):
     def __init__(self, reason: Any = None, message: str = "", code: int = -1) -> None:
         self.reason = reason  # 异常主要原因
         self.message = message  # 更细节的描述
-        self.code = code    # 错误码：-1 为默认；0～1000 以内正数为 http 错误码；10000 以上为自定义错误码
+        self.code = code  # 错误码：-1 为默认；0～1000 以内正数为 http 错误码；10000 以上为自定义错误码
 
     def __str__(self):
         return f"{self.__class__.__name__}: [{self.code}] {self.reason} {self.message}"
@@ -38,7 +38,7 @@ def __init__(self, message: str = ""):
 
 class ResourceNotFoundException(SelfDefinedException):
     def __init__(self, message: str = ""):
-        super().__init__(reason="errors.resourceNotFound", message=message)
+        super().__init__(reason="errors.resourceNotFound", message=message, code=404)
 
 
 class InvalidRequestException(SelfDefinedException):
@@ -69,3 +69,9 @@ def __init__(self, message: str = "", code: int = -1):
 class OpenaiApiException(OpenaiException):
     def __init__(self, message: str = "", code: int = -1):
         super().__init__(reason="errors.openaiWeb", message=message, code=code)
+
+
+class ArkoseForwardException(Exception):
+    def __init__(self, message: str = "", code: int = 404):
+        self.message = message
+        self.code = code

backend/api/response.py

@@ -11,7 +11,7 @@
 from starlette.background import BackgroundTask
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
-from api.exceptions import SelfDefinedException
+from api.exceptions import SelfDefinedException, ArkoseForwardException
 from utils.common import desensitize
 
 T = TypeVar('T')
@@ -102,3 +102,7 @@ def handle_exception_response(e: Exception) -> CustomJSONResponse:
             tip = get_http_message(e.status_code)
         return response(e.status_code or -1, tip, desensitize(f"{e.status_code} {e.detail}"))
     return response(-1, desensitize(f"{e.__class__.__name__}: {desensitize(str(e))}"))
+
+
+def handle_arkose_forward_exception(e: ArkoseForwardException):
+    return Response(content=e.message, status_code=e.code, media_type="application/plain")

backend/api/routers/arkose.py

@@ -0,0 +1,39 @@
+import httpx
+from fastapi import APIRouter, Depends, Response
+
+from api.conf import Config
+from api.exceptions import ResourceNotFoundException, ArkoseForwardException
+from api.models.db import User
+from api.sources import OpenaiWebChatManager
+from api.users import current_active_user
+
+config = Config()
+router = APIRouter()
+openai_web_manager = OpenaiWebChatManager()
+
+
+@router.get("/arkose/v2/{path:path}", tags=["arkose"])
+async def forward_arkose_request(path: str, _user: User = Depends(current_active_user)):
+    """
+    TODO 经过转发，arkose 会报错 "API_REQUEST_ERROR"
+    """
+    try:
+        resp = await openai_web_manager.session.get(f"{config.openai_web.arkose_endpoint_base}{path}")
+        resp.raise_for_status()
+        headers = dict(resp.headers)
+        media_type = resp.headers.get("content-type")
+        headers.pop("content-length", None)
+        headers.pop("content-encoding", None)
+        return Response(content=resp.content, media_type=media_type, headers=headers)
+    except httpx.HTTPStatusError as e:
+        raise ArkoseForwardException(code=e.response.status_code, message=e.response.text)
+    except Exception as e:
+        return ArkoseForwardException(code=500, message=str(e))
+
+
+@router.get("/arkose/info", tags=["arkose"])
+async def get_arkose_info(_user: User = Depends(current_active_user)):
+    return {
+        "enabled": config.openai_web.enable_arkose_endpoint,
+        "url": f"{config.openai_web.arkose_endpoint_base}35536E1E-65B4-4D96-9D97-6ADB7EFF8147/api.js"
+    }

backend/api/routers/chat.py

@@ -361,6 +361,7 @@ async def reply(response: AskResponse):
                                            plugin_ids=ask_request.openai_web_plugin_ids if ask_request.new_conversation else None,
                                            attachments=ask_request.openai_web_attachments,
                                            multimodal_image_parts=ask_request.openai_web_multimodal_image_parts,
+                                           arkose_token=ask_request.arkose_token,
                                            ):
             has_got_reply = True
 

backend/api/schemas/conversation_schemas.py

@@ -41,6 +41,7 @@ class AskRequest(BaseModel):
     openai_web_plugin_ids: Optional[list[str]] = None
     openai_web_attachments: Optional[list[OpenaiWebChatMessageMetadataAttachment]] = None
     openai_web_multimodal_image_parts: Optional[list[OpenaiWebChatMessageMultimodalTextContentImagePart]] = None
+    arkose_token: Optional[str] = None
 
     @model_validator(mode='before')
     @classmethod

backend/api/sources/openai_web.py

@@ -320,6 +320,7 @@ async def complete(self, model: OpenaiWebChatModels, text_content: str, use_team
                        plugin_ids: list[str] = None,
                        attachments: list[OpenaiWebChatMessageMetadataAttachment] = None,
                        multimodal_image_parts: list[OpenaiWebChatMessageMultimodalTextContentImagePart] = None,
+                       arkose_token: str = None,
                        **_kwargs):
 
         assert config.openai_web.enabled, "OpenAI Web is not enabled"
@@ -367,23 +368,27 @@ async def complete(self, model: OpenaiWebChatModels, text_content: str, use_team
 
         completion_request = OpenaiWebCompleteRequest(
             action=action,
-            arkose_token=None,
+            arkose_token=arkose_token,
             conversation_mode=OpenaiWebCompleteRequestConversationMode(kind="primary_assistant"),
             conversation_id=str(conversation_id) if conversation_id else None,
             messages=messages,
             parent_message_id=str(parent_message_id) if parent_message_id else None,
             model=model.code(),
             plugin_ids=plugin_ids
-        ).dict(exclude_none=True)
-        completion_request["arkose_token"] = None
+        )
+        completion_request_dict = completion_request.dict(exclude_none=True)
+        if "arkose_token" not in completion_request_dict:
+            completion_request_dict["arkose_token"] = None
         data_json = json.dumps(jsonable_encoder(completion_request))
 
+        headers = req_headers(use_team) | {
+            "referer": "https://chat.openai.com/" + (f"c/{conversation_id}" if conversation_id else "")}
+        if arkose_token is not None:
+            headers["Openai-Sentinel-Arkose-Token"] = arkose_token
+
         async with self.session.stream(method="POST", url=f"{config.openai_web.chatgpt_base_url}conversation",
                                        data=data_json, timeout=timeout,
-                                       headers=req_headers(use_team) | {
-                                           "referer": "https://chat.openai.com/" + (
-                                                   f"c/{conversation_id}" if conversation_id else "")
-                                       }) as response:
+                                       headers=headers) as response:
             await _check_response(response)
 
             async for line in response.aiter_lines():

backend/config_templates/config.yaml

@@ -6,6 +6,8 @@ openai_web:
   chatgpt_base_url:
   proxy:
   wss_proxy:
+  enable_arkose_endpoint: false
+  arkose_endpoint_base:
   common_timeout: 20
   ask_timeout: 600
   sync_conversations_on_startup: false

backend/main.py

@@ -19,11 +19,11 @@
 from api.database.sqlalchemy import initialize_db, get_async_session_context, get_user_db_context
 from api.database.mongodb import init_mongodb
 from api.enums import OpenaiWebChatStatus
-from api.exceptions import SelfDefinedException, UserAlreadyExists
+from api.exceptions import SelfDefinedException, UserAlreadyExists, ArkoseForwardException
 from api.middlewares import AccessLoggerMiddleware, StatisticsMiddleware
 from api.models.db import User
-from api.response import CustomJSONResponse, handle_exception_response
-from api.routers import users, conv, chat, system, status, files, logs
+from api.response import CustomJSONResponse, handle_exception_response, handle_arkose_forward_exception
+from api.routers import users, conv, chat, system, status, files, logs, arkose
 from api.schemas import UserCreate, UserSettingSchema
 from api.sources import OpenaiWebChatManager
 from api.users import get_user_manager_context
@@ -129,8 +129,8 @@ async def lifespan(app: FastAPI):
 app.include_router(logs.router)
 app.include_router(status.router)
 app.include_router(files.router)
+app.include_router(arkose.router)
 
-# 解决跨站问题
 app.add_middleware(
     CORSMiddleware,
     allow_origins=config.http.cors_allow_origins,
@@ -151,10 +151,15 @@ async def validation_exception_handler(request, exc):
 
 
 @app.exception_handler(SelfDefinedException)
-async def validation_exception_handler(request, exc):
+async def self_defined_exception_handler(request, exc):
     return handle_exception_response(exc)
 
 
+@app.exception_handler(ArkoseForwardException)
+async def arkose_forward_exception_handler(request, exc):
+    return handle_arkose_forward_exception(exc)
+
+
 if __name__ == "__main__":
     uvicorn.run(app, host=config.http.host,
                 port=config.http.port,

frontend/config/vite.config.dev.ts

@@ -14,7 +14,7 @@ export default mergeConfig(
       proxy: {
         '/api': {
           target: 'http://127.0.0.1:8000',
-          changeOrigin: true,
+          changeOrigin: false,
           ws: true,
           rewrite: (path) => path.replace(/^\/api/, ''),
         },

frontend/index.html

@@ -5,6 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
     <link rel="icon" type="image/png" href="/icon.png" />
     <link rel="code-repository" href="https://github.com/chatpire/chatgpt-web-share" />
+    <!-- <script id="arkose-test" type="text/javascript" src="http://localhost:8000/arkose/api.js" data-callback="setupEnforcement" async defer></script> -->
     <title>ChatGPT Web Share</title>
   </head>
   <body>

frontend/src/api/arkose.ts

@@ -0,0 +1,7 @@
+import axios from 'axios';
+
+import ApiUrl from './url';
+
+export function getArkoseInfo() {
+  return axios.get<{ enabled: boolean, url: string }>(ApiUrl.ArkoseInfo);
+}

frontend/src/api/url.ts

@@ -32,6 +32,8 @@ enum ApiUrl {
   FilesOpenaiWebUploadStart = '/files/openai-web/upload-start',
   FilesOpenaiWebUploadComplete = '/files/openai-web/upload-complete',
   FilesLocalUploadToOpenaiWeb = '/files/local/upload-to-openai-web',
+
+  ArkoseInfo = '/arkose/info',
 }
 
 export default ApiUrl;

frontend/src/locales/en-US.json

@@ -191,7 +191,8 @@
     "check_chatgpt_accounts": "Check ChatGPT Account Availability",
     "limit": "limit",
     "timeRange": "Time Range",
-    "user": "user"
+    "user": "user",
+    "test_arkose": "Test Arkose"
   },
   "errors": {
     "403": "403 error: access denied",
@@ -311,7 +312,8 @@
     "sandboxFileNotFound": "File {0} not found, either because the sandbox path is wrong or because the interpreter is expired.",
     "dragFileOrImageHere": "Click or drag any file here to upload",
     "gpt4UploadingRequirements": "No more than 10 files or images, each file no larger than 512MB. Please ensure the files do not contain sensitive information.",
-    "check_chatgpt_accounts": "Please use this feature after saving the configuration and updating to the latest access token. If you have a Team subscription, the Team Account ID will be filled in automatically."
+    "check_chatgpt_accounts": "Please use this feature after saving the configuration and updating to the latest access token. If you have a Team subscription, the Team Account ID will be filled in automatically.",
+    "test_arkose": "Please refer to the documentation to set arkose_endpoint_base in config and save it before using."
   },
   "labels": {
     "nickname": "Nickname",

frontend/src/locales/zh-CN.json

@@ -191,7 +191,8 @@
     "time": "时间",
     "timeRange": "时间范围",
     "limit": "限制",
-    "check_chatgpt_accounts": "检查 ChatGPT 账户可用性"
+    "check_chatgpt_accounts": "检查 ChatGPT 账户可用性",
+    "test_arkose": "测试 Arkose"
   },
   "errors": {
     "403": "403 错误：无访问权限",
@@ -238,7 +239,9 @@
       "unknown": "未知 OpenAI 错误"
     },
     "unknown": "未知错误",
-    "invalidRequest": "非法请求"
+    "invalidRequest": "非法请求",
+    "arkoseError": "Arkose 验证失败",
+    "arkoseTokenError": "Arkose 验证失败，请联系管理员处理。"
   },
   "tips": {
     "loginExpired": "登录已过期。是否跳转到登录页面？",
@@ -313,7 +316,8 @@
     "gpt4UploadingRequirements": "不超过10份文件或图片，每份文件不超过512MB；请确保文件不含敏感信息",
     "check_chatgpt_accounts": "请在保存配置并更新到最新 Access Token 后使用此功能。如果您开通了 Team 订阅，将会自动填写 Team Account ID。",
     "autoFillTeamAccountIdSuccess": "已自动填写 Team Account ID",
-    "checkChatgptAccountsFailed": "请求失败"
+    "checkChatgptAccountsFailed": "请求失败",
+    "test_arkose": "请参考文档填写 arkose_endpoint_base 设置并保存后使用此功能"
   },
   "labels": {
     "nickname": "昵称",
@@ -357,7 +361,8 @@
       "enabled_models": "启用的模型",
       "model_code_mapping": "模型代码映射",
       "file_upload_strategy": "文件上传策略",
-      "max_completion_concurrency": "最大并行对话数量"
+      "max_completion_concurrency": "最大并行对话数量",
+      "enable_arkose_endpoint": "启用前端 Arkose 验证"
     },
     "conversation_id": "对话ID",
     "queueing_time": "排队时长",
@@ -404,7 +409,9 @@
     "config": {
       "enabled_models": "设置全局启用哪些模型，在此处不启用的模型所有账号都无法使用",
       "file_upload_strategy": "配置使用 code interpreter 模型时文件如何上传，详见文档",
-      "max_completion_concurrency": "决定同时能有多少用户进行对话。如果设置为一个很大的值，相当于禁用排队对话功能。"
+      "max_completion_concurrency": "决定同时能有多少用户进行对话。如果设置为一个很大的值，相当于禁用排队对话功能。",
+      "enable_arkose_endpoint": "实验性功能，启用前请仔细阅读文档说明。启用后务必填写下方的 arkose_endpoint_base。如果启用，将会在前端取得 Arkose token，用户可能需要进行手动验证。",
+      "arkose_endpoint_base": "Arkose 代理的地址前缀，必须以 /v2/ 结尾，如：http://ninja/v2/"
     }
   },
   "dialog": {

frontend/src/types/json/config_schema.json

@@ -278,6 +278,23 @@
           "default": null,
           "title": "Wss Proxy"
         },
+        "enable_arkose_endpoint": {
+          "default": false,
+          "title": "Enable Arkose Endpoint",
+          "type": "boolean"
+        },
+        "arkose_endpoint_base": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Arkose Endpoint Base"
+        },
         "common_timeout": {
           "default": 20,
           "description": "Increase this value if timeout error occurs.",
@@ -424,6 +441,8 @@
         "chatgpt_base_url": null,
         "proxy": null,
         "wss_proxy": null,
+        "enable_arkose_endpoint": false,
+        "arkose_endpoint_base": null,
         "common_timeout": 20,
         "ask_timeout": 600,
         "sync_conversations_on_startup": false,