Merge branch 'main' of github.com:1Panel-dev/MaxKB into pr@main@fix_bugs

apps/application/chat_pipeline/step/chat_step/i_chat_step.py

@@ -44,7 +44,7 @@ class PostResponseHandler:
     @abstractmethod
     def handler(self, chat_id, chat_record_id, paragraph_list: List[ParagraphPipelineModel], problem_text: str,
                 answer_text,
-                manage, step, padding_problem_text: str = None, **kwargs):
+                manage, step, padding_problem_text: str = None, client_id=None, **kwargs):
         pass
 
 

apps/application/chat_pipeline/step/chat_step/impl/base_chat_step.py

@@ -67,7 +67,7 @@ def event_content(response,
         manage.context['message_tokens'] = manage.context['message_tokens'] + request_token
         manage.context['answer_tokens'] = manage.context['answer_tokens'] + response_token
         post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
-                                      all_text, manage, step, padding_problem_text)
+                                      all_text, manage, step, padding_problem_text, client_id)
         yield 'data: ' + json.dumps({'chat_id': str(chat_id), 'id': str(chat_record_id), 'operate': True,
                                      'content': '', 'is_end': True}) + "\n\n"
         add_access_num(client_id, client_type)
@@ -172,7 +172,7 @@ def execute_block(self, message_list: List[BaseMessage],
         manage.context['message_tokens'] = manage.context['message_tokens'] + request_token
         manage.context['answer_tokens'] = manage.context['answer_tokens'] + response_token
         post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
-                                      chat_result.content, manage, self, padding_problem_text)
+                                      chat_result.content, manage, self, padding_problem_text, client_id)
         add_access_num(client_id, client_type)
         return result.success({'chat_id': str(chat_id), 'id': str(chat_record_id), 'operate': True,
                                'content': chat_result.content, 'is_end': True})

apps/application/migrations/0002_chat_client_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.13 on 2024-03-28 13:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('application', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chat',
+            name='client_id',
+            field=models.UUIDField(default=None, null=True, verbose_name='客户端id'),
+        ),
+    ]

apps/application/models/application.py

@@ -69,6 +69,7 @@ class Chat(AppModelMixin):
     id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid1, editable=False, verbose_name="主键id")
     application = models.ForeignKey(Application, on_delete=models.CASCADE)
     abstract = models.CharField(max_length=256, verbose_name="摘要")
+    client_id = models.UUIDField(verbose_name="客户端id", default=None, null=True)
 
     class Meta:
         db_table = "application_chat"

apps/application/serializers/application_statistics_serializers.py

@@ -0,0 +1,128 @@
+# coding=utf-8
+"""
+    @project: maxkb
+    @Author：虎
+    @file： application_statistics_serializers.py
+    @date：2024/3/27 10:55
+    @desc:
+"""
+import datetime
+import os
+from typing import List, Dict
+
+from django.db import models
+from django.db.models.query import QuerySet
+from rest_framework import serializers
+
+from application.models.api_key_model import ApplicationPublicAccessClient
+from common.db.search import native_search, get_dynamics_model
+from common.util.field_message import ErrMessage
+from common.util.file_util import get_file_content
+from smartdoc.conf import PROJECT_DIR
+
+
+class ApplicationStatisticsSerializer(serializers.Serializer):
+    application_id = serializers.UUIDField(required=True, error_messages=ErrMessage.char("应用id"))
+    start_time = serializers.DateField(format='%Y-%m-%d', error_messages=ErrMessage.date("开始时间"))
+    end_time = serializers.DateField(format='%Y-%m-%d', error_messages=ErrMessage.date("结束时间"))
+
+    def get_end_time(self):
+        return datetime.datetime.combine(
+            datetime.datetime.strptime(self.data.get('end_time'), '%Y-%m-%d'),
+            datetime.datetime.max.time())
+
+    def get_start_time(self):
+        return self.data.get('start_time')
+
+    def get_customer_count(self, with_valid=True):
+        if with_valid:
+            self.is_valid(raise_exception=True)
+        start_time = self.get_start_time()
+        end_time = self.get_end_time()
+        return native_search(
+            QuerySet(ApplicationPublicAccessClient).filter(application_id=self.data.get('application_id'),
+                                                           create_time__gte=start_time,
+                                                           create_time__lte=end_time),
+            select_string=get_file_content(
+                os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'customer_count.sql')),
+            with_search_one=True)
+
+    def get_customer_count_trend(self, with_valid=True):
+        if with_valid:
+            self.is_valid(raise_exception=True)
+        start_time = self.get_start_time()
+        end_time = self.get_end_time()
+        return native_search(
+            {'default_sql': QuerySet(ApplicationPublicAccessClient).filter(
+                application_id=self.data.get('application_id'),
+                create_time__gte=start_time,
+                create_time__lte=end_time)},
+            select_string=get_file_content(
+                os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'customer_count_trend.sql')))
+
+    def get_chat_record_aggregate(self, with_valid=True):
+        if with_valid:
+            self.is_valid(raise_exception=True)
+        start_time = self.get_start_time()
+        end_time = self.get_end_time()
+        chat_record_aggregate = native_search(
+            QuerySet(model=get_dynamics_model(
+                {'application_chat.application_id': models.UUIDField(),
+                 'application_chat_record.create_time': models.DateTimeField()})).filter(
+                **{'application_chat.application_id': self.data.get('application_id'),
+                   'application_chat_record.create_time__gte': start_time,
+                   'application_chat_record.create_time__lte': end_time}
+            ),
+            select_string=get_file_content(
+                os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'chat_record_count.sql')),
+            with_search_one=True)
+        customer = self.get_customer_count(with_valid=False)
+        return {**chat_record_aggregate, **customer}
+
+    def get_chat_record_aggregate_trend(self, with_valid=True):
+        if with_valid:
+            self.is_valid(raise_exception=True)
+        start_time = self.get_start_time()
+        end_time = self.get_end_time()
+        chat_record_aggregate_trend = native_search(
+            {'default_sql': QuerySet(model=get_dynamics_model(
+                {'application_chat.application_id': models.UUIDField(),
+                 'application_chat_record.create_time': models.DateTimeField()})).filter(
+                **{'application_chat.application_id': self.data.get('application_id'),
+                   'application_chat_record.create_time__gte': start_time,
+                   'application_chat_record.create_time__lte': end_time}
+            )},
+            select_string=get_file_content(
+                os.path.join(PROJECT_DIR, "apps", "application", 'sql', 'chat_record_count_trend.sql')))
+        customer_count_trend = self.get_customer_count_trend(with_valid=False)
+        return self.merge_customer_chat_record(chat_record_aggregate_trend, customer_count_trend)
+
+    def merge_customer_chat_record(self, chat_record_aggregate_trend: List[Dict], customer_count_trend: List[Dict]):
+
+        return [{**self.find(chat_record_aggregate_trend, lambda c: c.get('day').strftime('%Y-%m-%d') == day,
+                             {'star_num': 0, 'trample_num': 0, 'tokens_num': 0, 'chat_record_count': 0,
+                              'customer_num': 0,
+                              'day': day}),
+                 **self.find(customer_count_trend, lambda c: c.get('day').strftime('%Y-%m-%d') == day,
+                             {'customer_added_count': 0})}
+                for
+                day in
+                self.get_days_between_dates(self.data.get('start_time'), self.data.get('end_time'))]
+
+    @staticmethod
+    def find(source_list, condition, default):
+        value_list = [row for row in source_list if condition(row)]
+        if len(value_list) > 0:
+            return value_list[0]
+        return default
+
+    @staticmethod
+    def get_days_between_dates(start_date, end_date):
+        start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
+        end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
+        days = []
+        current_date = start_date
+        while current_date <= end_date:
+            days.append(current_date.strftime('%Y-%m-%d'))
+            current_date += datetime.timedelta(days=1)
+        return days

apps/application/serializers/chat_message_serializers.py

@@ -87,13 +87,14 @@ def to_pipeline_manage_params(self, problem_text: str, post_response_handler: Po
                 'exclude_paragraph_id_list': exclude_paragraph_id_list, 'stream': stream, 'client_id': client_id,
                 'client_type': client_type}
 
-    def append_chat_record(self, chat_record: ChatRecord):
+    def append_chat_record(self, chat_record: ChatRecord, client_id=None):
         # 存入缓存中
         self.chat_record_list.append(chat_record)
         if self.application.id is not None:
             # 插入数据库
             if not QuerySet(Chat).filter(id=self.chat_id).exists():
-                Chat(id=self.chat_id, application_id=self.application.id, abstract=chat_record.problem_text).save()
+                Chat(id=self.chat_id, application_id=self.application.id, abstract=chat_record.problem_text,
+                     client_id=client_id).save()
             # 插入会话记录
             chat_record.save()
 
@@ -110,6 +111,7 @@ def handler(self,
                     manage: PiplineManage,
                     step: BaseChatStep,
                     padding_problem_text: str = None,
+                    client_id=None,
                     **kwargs):
             chat_record = ChatRecord(id=chat_record_id,
                                      chat_id=chat_id,
@@ -120,7 +122,7 @@ def handler(self,
                                      answer_tokens=manage.context['answer_tokens'],
                                      run_time=manage.context['run_time'],
                                      index=len(chat_info.chat_record_list) + 1)
-            chat_info.append_chat_record(chat_record)
+            chat_info.append_chat_record(chat_record, client_id)
             # 重新设置缓存
             chat_cache.set(chat_id,
                            chat_info, timeout=60 * 30)

apps/application/sql/chat_record_count.sql

@@ -0,0 +1,9 @@
+SELECT SUM
+	( CASE WHEN application_chat_record.vote_status = '0' THEN 1 ELSE 0 END ) AS "star_num",
+	SUM ( CASE WHEN application_chat_record.vote_status = '1' THEN 1 ELSE 0 END ) AS "trample_num",
+	SUM ( application_chat_record.message_tokens + application_chat_record.answer_tokens ) as "tokens_num",
+	"count"(DISTINCT application_chat.client_id) customer_num,
+	"count"(application_chat_record."id") as chat_record_count
+FROM
+	application_chat_record application_chat_record
+	LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id

apps/application/sql/chat_record_count_trend.sql

@@ -0,0 +1,12 @@
+SELECT SUM
+	( CASE WHEN application_chat_record.vote_status = '0' THEN 1 ELSE 0 END ) AS "star_num",
+	SUM ( CASE WHEN application_chat_record.vote_status = '1' THEN 1 ELSE 0 END ) AS "trample_num",
+	SUM ( application_chat_record.message_tokens + application_chat_record.answer_tokens ) as "tokens_num",
+	"count"(application_chat_record."id") as chat_record_count,
+	"count"(DISTINCT application_chat.client_id) customer_num,
+	application_chat_record.create_time :: DATE as "day"
+FROM
+	application_chat_record application_chat_record
+	LEFT JOIN application_chat application_chat ON application_chat."id" = application_chat_record.chat_id
+${default_sql}
+GROUP BY "day"

apps/application/sql/customer_count.sql

@@ -0,0 +1,5 @@
+SELECT
+	( SUM ( CASE WHEN create_time :: DATE = CURRENT_DATE THEN 1 ELSE 0 END ) ) AS "customer_today_added_count",
+	COUNT ( "application_public_access_client"."id" ) AS "customer_added_count"
+FROM
+	"application_public_access_client"

apps/application/sql/customer_count_trend.sql

@@ -0,0 +1,7 @@
+SELECT
+	COUNT ( "application_public_access_client"."id" ) AS "customer_added_count",
+	create_time :: DATE as "day"
+FROM
+	"application_public_access_client"
+${default_sql}
+GROUP BY "day"

apps/application/swagger_api/application_statistics_api.py

@@ -0,0 +1,86 @@
+# coding=utf-8
+"""
+    @project: maxkb
+    @Author：虎
+    @file： application_statistics_api.py
+    @date：2024/3/27 15:09
+    @desc:
+"""
+from drf_yasg import openapi
+
+from common.mixins.api_mixin import ApiMixin
+
+
+class ApplicationStatisticsApi(ApiMixin):
+    @staticmethod
+    def get_request_params_api():
+        return [openapi.Parameter(name='application_id',
+                                  in_=openapi.IN_PATH,
+                                  type=openapi.TYPE_STRING,
+                                  required=True,
+                                  description='应用id'),
+                openapi.Parameter(name='start_time',
+                                  in_=openapi.IN_QUERY,
+                                  type=openapi.TYPE_STRING,
+                                  required=True,
+                                  description='开始时间'),
+                openapi.Parameter(name='end_time',
+                                  in_=openapi.IN_QUERY,
+                                  type=openapi.TYPE_STRING,
+                                  required=True,
+                                  description='结束时间'),
+                ]
+
+    class ChatRecordAggregate(ApiMixin):
+        @staticmethod
+        def get_response_body_api():
+            return openapi.Schema(
+                type=openapi.TYPE_OBJECT,
+                required=['star_num', 'trample_num', 'tokens_num', 'chat_record_count'],
+                properties={
+                    'star_num': openapi.Schema(type=openapi.TYPE_NUMBER, title="点赞数量",
+                                               description="点赞数量"),
+
+                    'trample_num': openapi.Schema(type=openapi.TYPE_NUMBER, title="点踩数量", description="点踩数量"),
+                    'tokens_num': openapi.Schema(type=openapi.TYPE_NUMBER, title="token使用数量",
+                                                 description="token使用数量"),
+                    'chat_record_count': openapi.Schema(type=openapi.TYPE_NUMBER, title="对话次数",
+                                                        description="对话次数"),
+                    'customer_num': openapi.Schema(type=openapi.TYPE_NUMBER, title="客户数量",
+                                                   description="客户数量"),
+                    'customer_added_count': openapi.Schema(type=openapi.TYPE_NUMBER, title="客户新增数量",
+                                                           description="客户新增数量"),
+                    'day': openapi.Schema(type=openapi.TYPE_STRING,
+                                          title="日期",
+                                          description="日期,只有查询趋势的时候才有该字段"),
+                }
+            )
+
+    class CustomerCountTrend(ApiMixin):
+        @staticmethod
+        def get_response_body_api():
+            return openapi.Schema(
+                type=openapi.TYPE_OBJECT,
+                required=['added_count'],
+                properties={
+                    'added_count': openapi.Schema(type=openapi.TYPE_NUMBER, title="新增数量", description="新增数量"),
+
+                    'day': openapi.Schema(type=openapi.TYPE_STRING,
+                                          title="时间",
+                                          description="时间"),
+                }
+            )
+
+    class CustomerCount(ApiMixin):
+        @staticmethod
+        def get_response_body_api():
+            return openapi.Schema(
+                type=openapi.TYPE_OBJECT,
+                required=['added_count'],
+                properties={
+                    'today_added_count': openapi.Schema(type=openapi.TYPE_NUMBER, title="今日新增数量",
+                                                        description="今日新增数量"),
+                    'added_count': openapi.Schema(type=openapi.TYPE_NUMBER, title="新增数量", description="新增数量"),
+
+                }
+            )

apps/application/urls.py

@@ -8,6 +8,14 @@
     path('application/profile', views.Application.Profile.as_view()),
     path('application/embed', views.Application.Embed.as_view()),
     path('application/authentication', views.Application.Authentication.as_view()),
+    path('application/<str:application_id>/statistics/customer_count',
+         views.ApplicationStatistics.CustomerCount.as_view()),
+    path('application/<str:application_id>/statistics/customer_count_trend',
+         views.ApplicationStatistics.CustomerCountTrend.as_view()),
+    path('application/<str:application_id>/statistics/chat_record_aggregate',
+         views.ApplicationStatistics.ChatRecordAggregate.as_view()),
+    path('application/<str:application_id>/statistics/chat_record_aggregate_trend',
+         views.ApplicationStatistics.ChatRecordAggregateTrend.as_view()),
     path('application/<str:application_id>/model', views.Application.Model.as_view()),
     path('application/<str:application_id>/hit_test', views.Application.HitTest.as_view()),
     path('application/<str:application_id>/api_key', views.Application.ApplicationKey.as_view()),