fix: 【知识库】导入非utf8 编码的txt文件，分段内容是空白

apps/common/util/split_model.py

@@ -357,7 +357,7 @@ def filter_title_special_characters(paragraph: Dict):
 default_split_pattern = {
     'md': [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile('(?<!#)## (?!#).*'), re.compile("(?<!#)### (?!#).*"),
            re.compile("(?<!#)#### (?!#).*"), re.compile("(?<!#)##### (?!#).*"),
-           re.compile("(?<!#)###### (?!#).*")],
+           re.compile("(?<!#)###### (?!#).*"), re.compile("(?<!\n)\n\n+")],
     'default': [re.compile("(?<!\n)\n\n+")]
 }
 
@@ -374,7 +374,7 @@ def get_split_model(filename: str, with_filter: bool = False, limit: int = 4096)
         pattern_list = default_split_pattern.get('md')
         return SplitModel(pattern_list, with_filter=with_filter, limit=limit)
 
-    pattern_list = default_split_pattern.get('default')
+    pattern_list = default_split_pattern.get('md')
     return SplitModel(pattern_list, with_filter=with_filter, limit=limit)
 
 

apps/dataset/serializers/document_serializers.py

@@ -32,6 +32,7 @@
 from dataset.serializers.common_serializers import BatchSerializer, MetaSerializer
 from dataset.serializers.paragraph_serializers import ParagraphSerializers, ParagraphInstanceSerializer
 from smartdoc.conf import PROJECT_DIR
+import chardet
 
 
 class DocumentEditInstanceSerializer(ApiMixin, serializers.Serializer):
@@ -599,7 +600,7 @@ def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int):
     else:
         split_model = get_split_model(file.name, with_filter=with_filter, limit=limit)
     try:
-        content = data.decode('utf-8')
+        content = data.decode(chardet.detect(data)['encoding'])
     except BaseException as e:
         return {'name': file.name,
                 'content': []}

pyproject.toml

@@ -30,6 +30,7 @@ html2text = "^2024.2.26"
 langchain-openai = "^0.0.8"
 django-ipware = "^6.0.4"
 django-apscheduler = "^0.6.2"
+chardet2 = "^2.0.3"
 
 [build-system]
 requires = ["poetry-core"]