diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 36834f3a0..39457a1a7 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -4,7 +4,8 @@ on: push: branches: - 'main' - - 'develop-[0-9].[0-9].[0-9]' + - 'develop-[0-9]+.[0-9]+.[0-9]+' + - 'build-doc-[0-9]+.[0-9]+.[0-9]+-[a-zA-Z]+' schedule: - cron: '0 8 * * *' @@ -41,6 +42,7 @@ jobs: VERSION='${{ github.ref_name }}' [ "$VERSION" == main ] && { VERSION=latest; ALIAS='main master'; } VERSION="${VERSION#develop-}" + VERSION="${VERSION#build-doc-}" mike deploy --push --update-aliases "$VERSION" $ALIAS mike set-default --push latest diff --git a/.gitignore b/.gitignore index 6f4d52212..44e3d77a5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,12 +17,11 @@ venv /logs/ /jobs/ /audit/ +/localfs/ .vscode/* /temp/ /tmp /worker/ -/provider_registrar/ -/model_local_cache/ *.db *.db-journal *.whl @@ -32,3 +31,14 @@ venv # doc /site/ + +/python/fate_flow/data +/python/fate_flow/model +/python/fate_flow/logs +/python/fate_flow/jobs +/python/fate_flow/localfs +/python/fate_flow/*.env +/python/fate_flow/conf +/python/build +/python/dist +/python/*.egg-info \ No newline at end of file diff --git a/README.md b/README.md index 95cf06702..25ea2995a 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,6 @@ Providing production-level service capabilities: - High Availability - CLI, REST API, Python API -For detailed introduction, please refer to [FATE Flow Overall Design](https://federatedai.github.io/FATE-Flow/latest/fate_flow/#overall-design) - ## Deployment Please refer to [FATE](https://github.com/FederatedAI/FATE) diff --git a/README.zh.md b/README.zh.md index b1aaaece9..13da9e4cb 100644 --- a/README.zh.md +++ b/README.zh.md @@ -4,7 +4,7 @@ FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调度平台, 基于: -- [共享状态调度架构](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/41684.pdf) +- 共享状态调度架构 - 跨数据中心的多方安全通信 提供生产级服务能力: @@ -20,7 +20,6 @@ FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调 - 系统高可用 - CLI、REST API、Python API -详细介绍请参考[FATE Flow整体设计](https://federatedai.github.io/FATE-Flow/latest/zh/fate_flow/) ## 部署 diff --git a/RELEASE.md b/RELEASE.md index 422623e48..7530fbd6e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,14 @@ +## Release 2.0.0-beta +### Major Features and Improvements +* Migrated functions: data upload/download, process scheduling, component output data/model/metric management, multi-storage adaptation for models, authentication, authorization, feature anonymization, multi-computing/storage/communication engine adaptation, and system high availability +* Optimized process scheduling, with scheduling separated and customizable, and added priority scheduling +* Optimized algorithm component scheduling, dividing execution steps into preprocessing, running, and post-processing +* Optimized multi-version algorithm component registration, supporting registration for mode of components +* Optimized client authentication logic, supporting permission management for multiple clients +* Optimized RESTful interface, making parameter fields and types, return fields, and status codes clearer +* Decoupling the system layer from the algorithm layer, with system configuration moved from the FATE repository to the Flow repository +* Published FATE Flow package to PyPI and added service-level CLI for service management + ## Release 2.0.0-alpha ### Feature Highlights * Adapted to new scalable and standardized federated DSL IR diff --git a/bin/init_env.sh b/bin/init_env.sh index ce871e992..4fe16242f 100644 --- a/bin/init_env.sh +++ b/bin/init_env.sh @@ -18,10 +18,11 @@ fate_project_base=$(cd `dirname "$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"`; cd ../;cd ../;pwd) export FATE_PROJECT_BASE=$fate_project_base -export FATE_DEPLOY_BASE=$fate_project_base +export FATE_PYTHONPATH= export EGGROLL_HOME= export PYTHONPATH= export FATE_ENV= +export SPARK_HOME= export FATE_LOG_LEVEL=DEBUG diff --git a/bin/service.sh b/bin/service.sh index 3ec778c2c..5f213fd75 100644 --- a/bin/service.sh +++ b/bin/service.sh @@ -28,7 +28,7 @@ echo "PROJECT_BASE: "${PROJECT_BASE} INI_ENV_SCRIPT=${FATE_FLOW_BASE}/bin/init_env.sh echo $INI_ENV_SCRIPT if test -f "${INI_ENV_SCRIPT}"; then - source ${$INI_ENV_SCRIPT}/bin/init_env.sh + source $INI_ENV_SCRIPT echo "PYTHONPATH: "${PYTHONPATH} else echo "file not found: ${INI_ENV_SCRIPT}" diff --git a/conf/casbin_model.conf b/conf/casbin_model.conf new file mode 100644 index 000000000..71159e387 --- /dev/null +++ b/conf/casbin_model.conf @@ -0,0 +1,14 @@ +[request_definition] +r = sub, obj, act + +[policy_definition] +p = sub, obj, act + +[role_definition] +g = _, _ + +[policy_effect] +e = some(where (p.eft == allow)) + +[matchers] +m = g(r.sub, p.sub) && r.obj == p.obj && r.act == p.act \ No newline at end of file diff --git a/conf/job_default_config.yaml b/conf/job_default_config.yaml index e5cedf62d..c79e8b831 100644 --- a/conf/job_default_config.yaml +++ b/conf/job_default_config.yaml @@ -1,45 +1,27 @@ # resource -total_cores_overweight_percent: 1 # 1 means no overweight -total_memory_overweight_percent: 1 # 1 means no overweight -task_parallelism: 1 -task_cores: 4 -task_memory: 0 # mb -max_cores_percent_per_job: 1 # 1 means total - -# scheduling +job_cores: 4 +computing_partitions: 8 +task_run: + spark: + num-executors: 2 + executor-cores: 2 + eggroll: + eggroll.session.processors.per.node: 4 + standalone: + cores: 4 job_timeout: 259200 # s remote_request_timeout: 30000 # ms federated_command_trys: 3 -end_status_job_scheduling_time_limit: 300000 # ms -end_status_job_scheduling_updates: 1 -auto_retries: 1 -auto_retry_delay: 1 #seconds -# It can also be specified in the job configuration using the federated_status_collect_type parameter -federated_status_collect_type: PUSH -detect_connect_max_retry_count: 3 -detect_connect_long_retry_count: 2 - -# upload -upload_block_max_bytes: 104857600 # bytes - -#component output -output_data_summary_count_limit: 100 - -task_default_conf: - logger: - type: flow - metadata: - level: DEBUG - debug_mode: true - device: - type: CPU - output: - data: - type: directory - format: dataframe - model: - type: directory - format: json - metric: - type: directory - format: json +auto_retries: 0 +sync_type: callback # poll or callback +task_logger: + type: flow + metadata: + level: DEBUG + debug_mode: true +task_device: + type: CPU +launcher: + deepspeed: + timeout: 21600 # s + world_size: 2 diff --git a/conf/permission_casbin_model.conf b/conf/permission_casbin_model.conf new file mode 100644 index 000000000..60a4e14f9 --- /dev/null +++ b/conf/permission_casbin_model.conf @@ -0,0 +1,11 @@ +[request_definition] +r = party_id, type, value + +[policy_definition] +p = party_id, type, value + +[policy_effect] +e = some(where (p.eft == allow)) + +[matchers] +m = r.party_id == p.party_id && r.type == p.type && r.value == p.value \ No newline at end of file diff --git a/conf/pulsar_route_table.yaml b/conf/pulsar_route_table.yaml index 3783a8805..730b5001a 100644 --- a/conf/pulsar_route_table.yaml +++ b/conf/pulsar_route_table.yaml @@ -1,6 +1,6 @@ 9999: # host can be a domain like 9999.fate.org - host: 172.16.153.37 + host: 127.0.0.1 port: 6650 sslPort: 6651 # set proxy address for this pulsar cluster @@ -8,7 +8,7 @@ 10000: # host can be a domain like 10000.fate.org - host: 172.16.153.37 + host: 127.0.0.1 port: 6650 sslPort: 6651 proxy: "" diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml index 7acc99bb9..5c4fb4cb5 100644 --- a/conf/service_conf.yaml +++ b/conf/service_conf.yaml @@ -1,24 +1,44 @@ -force_use_sqlite: true -party_id: "10000" +party_id: "9999" +use_registry: false +encrypt: + key_0: + module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt + # base on: fate_flow/conf/ + private_path: private_key.pem fateflow: - # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported host: 127.0.0.1 http_port: 9380 grpc_port: 9360 proxy_name: rollsite + nginx: + host: + http_port: + grpc_port: database: - name: fate_flow - user: fate - passwd: fate - host: 127.0.0.1 - port: 3306 - max_connections: 100 - stale_timeout: 30 -# engine services + engine: sqlite + # encrypt passwd key + decrypt_key: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + sqlite: + # default fate_flow/runtime/system_settings: SQLITE_PATH + # /xxx/xxx.sqlite + path: default_engines: computing: standalone federation: standalone storage: standalone +default_provider: + name: fate + # version default: fateflow.env + version: + device: local federation: pulsar: host: 192.168.0.5 @@ -57,27 +77,48 @@ federation: port: 9370 computing: standalone: - cores_per_node: 20 - nodes: 1 + cores: 32 eggroll: - cores_per_node: 16 - nodes: 1 + cores: 32 + nodes: 2 spark: # default use SPARK_HOME environment variable home: - cores_per_node: 20 - nodes: 2 -worker: - type: native - docker: - config: - # https://docker-py.readthedocs.io/en/stable/client.html#docker.client.DockerClient - base_url: unix:///var/run/docker.sock - image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha - # on container - fate_root_dir: /data/projects/fate - # on host - eggroll_conf_dir: - k8s: - image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha - namespace: fate-10000 + cores: 32 +storage: + hdfs: + name_node: hdfs://fate-cluster +hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission +authentication: + client: false + site: false + permission: false +model_store: + engine: file + # encrypt passwd key + decrypt_key: + file: + # default fate_flow/runtime/system_settings: MODEL_STORE_PATH + path: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + tencent_cos: + Region: + SecretId: + SecretKey: + Bucket: +zookeeper: + hosts: + - 127.0.0.1:2181 + use_acl: true + user: fate + password: fate diff --git a/doc/2.0.0-alpha.md b/doc/2.0.0-alpha.md deleted file mode 100644 index 396c96b6a..000000000 --- a/doc/2.0.0-alpha.md +++ /dev/null @@ -1,83 +0,0 @@ -## FATE FLOW V2.0方案 - -### 1. 背景 - -联邦学习为打破“数据孤岛”而生，然而随着越来越多的机构投身到联邦学习领域，不同架构的联邦学习系统之间逐渐形成了新的“孤岛”现象，互联互通显得越发重要。FATE FLow 2.0版本将定义全新的Open Flow Api，从流程调度和算法调度两个层面实现互联互通。 - -### 2. 整体方案图 - -![image-20220922195625843](./images/open_flow.png) - -### 3. 调度层 - -#### 3.1 流程调度时序图 - -2.x版本调度方可在任务配置中指定(默认为发起方)，可以为发起方、合作方或者第三方。 - -##### 3.1.1 push模式 -说明：各参与方主动上报任务状态 -![image-20220922195625843](./images/push.png) - - -##### 3.1.2 pull模式 -说明：调度方定时查询各参与方任务状态 -![image-20220922195625843](./images/pull.png) - -#### 3.2 应用层 - -- 说明：用于对接上层系统，包括任务创建、查询、停止等接口 - -#### 3.3 底座层 - -- 说明：用于对接算法容器，包括任务状态上报、任务输出存储/查询等 - -#### 3.4 互联互通层 - -- 说明：用于对接跨机构、站点调度 - - -### 4. 算法容器调度 - -说明：FATE历史版本中的算法加载是以python脚本形式在subprocess进程中加载，在安全性、扩展性等方面存在不足，且无法满足异构算法组合编排场景。在新版本中引入“算法容器”加载算法，通过制定统一的算法镜像构建标准与接口并定义一套规范的镜像加载机制与流程，实现异构场景的互联互通。 - -![image-20220922195625843](./images/federationml_schedule.png) - -注：图中节点A、B代表两家隐私计算提供商，A-X代表A厂的算法X，B-Y代表B厂算法Y。 - -#### 4.1 容器注册与加载 - -- [算法容器注册与加载文档](./container.md) - -#### 4.2 平台资源 - -##### 4.2.1 通信 -新增支持osx(open site exchange)作为通信服务 -- 调度通信服务：rollsite、nginx、osx -- 算法通信服务：rollsite、rabbitmq、pulsar、osx - -##### 4.2.2 计算 - -- standalone -- eggroll -- spark - -##### 4.2.3 存储 - -- standalone -- eggroll -- hdfs -- ... - -### 5. DAG定义 -fate 2.0参考kubeflow的设计，在DAG的结构定义方面进行调整，具体参考: [新版dag配置](./../examples/lr/eggroll/lr_train_dag.yaml) - -### 6. 解耦 - -fate 1.x版本的调度层与算法层在数据、模型、类调用等方面存在一些耦合和特判的情况。在fate 2.0版本，在算法和调度层面解偶，以此降低异构算法接入的开发成本。 - -### 7. 资源管控 -- 资源类型: job、 task -- 管控粒度: job级资源控制任务数量、task级资源控制任务并行度 - -### 8. 状态码定义 -- 细化api状态码，以便快速定位问题 diff --git a/doc/build/build.py b/doc/build/build.py new file mode 100644 index 000000000..39eac2b4e --- /dev/null +++ b/doc/build/build.py @@ -0,0 +1,38 @@ +import json +import os.path +import subprocess +import sys +import threading + +import requests + + +def run_script(script_path, *args): + result = subprocess.run(['python', script_path, *args]) + return result.stderr + + +if __name__ == '__main__': + base_dir = os.path.dirname(__file__) + build_path = os.path.join(base_dir, 'build_swagger_server.py') + + thread = threading.Thread(target=run_script, args=(build_path,)) + thread.start() + # + thread.join() + build_path = os.path.join(base_dir, 'swagger_server.py') + port = "50000" + server = threading.Thread(target=run_script, args=(build_path, port)) + + result = server.start() + + import time + time.sleep(3) + data = requests.get(url=f"http://127.0.0.1:{port}/swagger.json").text + data = json.loads(data) + swagger_file = os.path.join(os.path.dirname(base_dir), "swagger", "swagger.json") + os.makedirs(os.path.dirname(swagger_file), exist_ok=True) + with open(swagger_file, "w") as fw: + json.dump(data, fw, indent=4) + print("build success!") + sys.exit() diff --git a/doc/build/build_swagger_server.py b/doc/build/build_swagger_server.py new file mode 100644 index 000000000..5c3990e15 --- /dev/null +++ b/doc/build/build_swagger_server.py @@ -0,0 +1,166 @@ +import ast +import os.path +import re +from importlib.util import spec_from_file_location, module_from_spec +from pathlib import Path + +import fate_flow +from fate_flow.runtime.system_settings import HOST, HTTP_PORT, API_VERSION + +base_path = f"/{API_VERSION}" +FATE_FLOW_HOME = os.path.dirname(fate_flow.__file__) +DOC_BASE = os.path.join(os.path.dirname(os.path.dirname(FATE_FLOW_HOME)), "doc", "build") +swagger_py_file = os.path.join(DOC_BASE, "swagger_server.py") + + +def search_pages_path(pages_dir): + return [path for path in pages_dir.glob('*_app.py') if not path.name.startswith('.')] + + +def read_desc_script(files): + with open(files, "r") as file: + content = file.read() + + pattern = r'(\w+)\s*=\s*"([^"]+)"' + variables = dict(re.findall(pattern, content)) + return variables + + +def scan_client_app(file_path, variables): + function_info = {} + for _path in file_path: + page_name = _path.stem.rstrip('app').rstrip("_") + module_name = '.'.join(_path.parts[_path.parts.index('apps') - 1:-1] + (page_name,)) + spec = spec_from_file_location(module_name, _path) + page = module_from_spec(spec) + page_name = getattr(page, 'page_name', page_name) + if page_name not in function_info: + function_info[page_name] = [] + with open(str(_path), 'r') as file: + tree = ast.parse(file.read()) + + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + function_name = node.name + function_params = [] + function_route = None + function_method = None + function_params_desc = {} + + for arg in node.args.args: + function_params.append(arg.arg) + + for decorator in node.decorator_list: + if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Attribute): + if decorator.func.attr == 'route': + function_route = decorator.args[0].s + if isinstance(decorator.keywords, list): + for keyword in decorator.keywords: + if keyword.arg == 'methods': + function_method = keyword.value.elts[0].s + + else: + params_value = "" + params_name = "" + for key in decorator.keywords: + if key.arg == 'desc': + params_value = key.value.id + else: + params_name = key.arg + + if params_name: + function_params_desc[params_name] = variables.get(params_value, "") + function_info[page_name].append({ + 'function_name': function_name, + 'function_route': function_route, + 'function_method': function_method, + 'function_params_desc': function_params_desc, + }) + + return function_info + + +def generate_transfer_doc(function_info): + script = f""" +from flask import Flask +from flask_restx import Api, Resource, Swagger +from werkzeug.utils import cached_property + + +class RSwagger(Swagger): + def as_dict_v2(self): + _dict = self.as_dict() + _dict["basePath"] = "{base_path}" + return _dict + + def operation_id_for(self, doc, method): + return ( + doc[method].get("operationId") + if "operationId" in doc[method] + else self.api.default_id(doc["name"], method) + ) + + def description_for(self, doc, method): + return doc[method].get("description") + + +class RApi(Api): + @cached_property + def __schema__(self): + if not self._schema: + try: + self._schema = RSwagger(self).as_dict_v2() + except Exception: + msg = "Unable to render schema" + log.exception(msg) + return msg + return self._schema + + +app = Flask(__name__) +api = RApi(app, version="{fate_flow.__version__}", title="FATE Flow restful api") +""" + + for page_name in function_info.keys(): + script += f""" +{page_name} = api.namespace("{page_name}", description="{page_name}-Related Operations") +""" + for page_name, infos in function_info.items(): + for info in infos: + function_name = ''.join([word.capitalize() for word in info['function_name'].split("_")]) + function_route = info['function_route'] + function_method = info['function_method'] + function_params_desc = info['function_params_desc'] + + script += f""" + +@{page_name}.route('{function_route}') +class {function_name}(Resource): + @api.doc(params={function_params_desc}, operationId='{function_method.lower()}_{page_name}_{function_name}', descrption='this is a test') + def {function_method.lower()}(self): + ''' + + ''' + # Your code here + return +""" + script += f""" + +if __name__ == '__main__': + import sys + if len(sys.argv) > 1: + port = int(sys.argv[1]) + else: + port = 5000 + app.run(port=port) +""" + return script + + +if __name__ == '__main__': + file_dir = search_pages_path(Path(FATE_FLOW_HOME) / 'apps/client') + variables = read_desc_script(Path(FATE_FLOW_HOME) / 'apps/desc.py') + function_info = scan_client_app(file_dir, variables) + transfer_doc_script = generate_transfer_doc(function_info) + with open(swagger_py_file, 'w', encoding='utf-8') as file: + file.write(transfer_doc_script) diff --git a/doc/container.md b/doc/container.md deleted file mode 100644 index 5b455a461..000000000 --- a/doc/container.md +++ /dev/null @@ -1,49 +0,0 @@ -## 算法容器注册与加载方案 - -### 1. 整体架构图 -![整体架构图](./images/container_load.png) - -模块说明： - -1. discovery: 算法服务发现和路由 -2. registry: 算法注册器，包括本地算法和算法镜像 -3. scheduler: 调度器，对任务进行调度 -4. local manager: 本地算法任务(非容器化方式)管理 -5. container manager: 基于容器化方式的管理，是对底层容器编排能力的封装 - -### 2. 容器加载 -#### 2.1 容器管理器 - -负责管理组件所运行的容器的管理，与容器平台(如：Docker、Kubernetes等)进行对接，完成容器的编排管理。 - -#### 2.2 容器运行模式 - -##### 2.2.1 即用即销毁 -说明：一个task对应一个容器 -实现方式：通过容器命令接口启动容器任务, 任务运行结束上报其对应状态和相应输出给系统，并销毁当前容器。 - -##### 2.2.2 常驻服务(待实现) -说明：支持容器复用，多个task运行在一个容器里 -实现方式：flow服务内嵌至算法容器中，并暴露相关接口(run、stop等)给调度层。调度层通过容器支持服务接口启动task。容器启动后为常驻服务，供调度层调度。 - -### 3. 参数传递方式 -#### 3.1 环境变量 -适用于即用即销毁模式，容器启动时将参数放到算法容器环境变量里。 -#### 3.2 接口参数(待实现) -适用于常驻服务模式，启动task的方式为调用容器服务接口，并把所需参数作为接口参数传入。 - -### 4. 算法注册与加载(待实现) -1. 注册：flow提供镜像注册接口(provider)，内容定义： -```yaml -provider: fate -version: 2.0.0.alpha -way: docker -``` -注：provider为算法来源;version为算法版本号;way为算法形式，包括local、docker、kubernetes等 -2. 加载：dag配置可指定provider，flow通过provider加载出对应算法模块； - -### 5. 容器日志方案 -![日志方案图](./images/log.png) -- 文件映射 -- 日志接口 -- 队列缓存+日志接口 diff --git a/doc/fate_flow.md b/doc/fate_flow.md new file mode 100644 index 000000000..9360d44db --- /dev/null +++ b/doc/fate_flow.md @@ -0,0 +1,110 @@ +# Overall Design + +## 1. Logical Architecture + +- DSL defined jobs +- Top-down vertical subtask flow scheduling, multi-participant joint subtask coordination +- Independent isolated task execution work processes +- Support for multiple types and versions of components +- Computational abstraction API +- Storage abstraction API +- Cross-party transfer abstraction API + +![](./images/fate_flow_logical_arch.png) + +## 2. Service Architecture + +### 2.1 FATE + +![](./images/fate_arch.png) + +### 2.2 FATE Flow + +![](./images/fate_flow_arch.png) + +## 3. [Scheduling Architecture](./fate_flow_job_scheduling.md) + +### 3.1 A new scheduling architecture based on shared-state + +- Stripping state (resources, jobs) and managers (schedulers, resource managers) +- Resource state and job state are persisted in MySQL and shared globally to provide reliable transactional operations +- Improve the high availability and scalability of managed services +- Jobs can be intervened to support restart, rerun, parallel control, resource isolation, etc. + +![](./images/fate_flow_scheduling_arch.png) + +### 3.2 State-Driven Scheduling + +- Resource coordination +- Pull up the child process Executor to run the component +- Executor reports state to local Server and also to scheduler +- Multi-party task state calculation of federal task state +- Upstream and downstream task states compute job states + +![](./images/fate_flow_resource_process.png) + +## 4. [Multiparty Resource Coordination](./fate_flow_resource_management.md) + +- The total resource size of each engine is configured through the configuration file, and the system is subsequently interfaced +- The cores_per_node in the total resource size indicates the number of cpu cores per compute node, and nodes indicates the number of compute nodes. +- FATEFlow server reads the resource size configuration from the configuration file when it starts and registers the update to the database +- The resources are requested in Job dimension, and take effect when Job Conf is submitted, formula: task_parallelism*task_cores +- See separate section of the documentation for details + +## 5. [Data Flow Tracking](./fate_flow_tracking.md) + +- Definition + - metric type: metric type, such as auc, loss, ks, etc. + - metric namespace: custom metric namespace, e.g. train, predict + - metric name: custom metric name, e.g. auc0, hetero_lr_auc0 + - metric data: metric data in key-value form + - metric meta: metric meta information in key-value form, support flexible drawing +- API + - log_metric_data(metric_namespace, metric_name, metrics) + - set_metric_meta(metric_namespace, metric_name, metric_meta) + - get_metric_data(metric_namespace, metric_name) + - get_metric_meta(metric_namespace, metric_name) + +## 6. [Realtime Monitoring](./fate_flow_monitoring.md) + +- Job process survivability detection +- Job timeout detection +- Resource recovery detection +- Base engine session timeout detection + +![](./images/fate_flow_detector.png) + +## 7. [Task Component Registry](./fate_flow_component_registry.md) + +![](./images/fate_flow_component_registry.png) + +## 8. [Multi-Party Federated Model Registry](./fate_flow_model_registry.md) + +- Using Google Protocol Buffer as the model storage protocol, using cross-language sharing, each algorithmic model consists of two parts: ModelParam & ModelMeta +- A Pipeline generates a series of algorithmic models +- The model named Pipeline stores Pipeline modeling DSL and online inference DSL +- Under federal learning, model consistency needs to be guaranteed for all participants, i.e., model binding +- model_key is the model identifier defined by the user when submitting the task +- The model IDs of the federated parties are the party identification information role, party_id, plus model_key +- The model version of the federated parties must be unique and consistent, and FATE-Flow directly sets it to job_id + +![](./images/fate_flow_pipelined_model.png){: style="height:400px;width:450px"} + +![](./images/fate_flow_model_storage.png){: style="height:400px;width:800px"} + +## 9. [Data Access](./fate_flow_data_access.md) + +- Upload. + - External storage is imported directly to FATE Storage, creating a new DTable + - When the job runs, Reader reads directly from Storage + +- Table Bind. + - Key the external storage address to a new DTable in FATE + - When the job is running, Reader reads data from external storage via Meta and transfers it to FATE Storage + - Connecting to the Big Data ecosystem: HDFS, Hive/MySQL + +![](./images/fate_flow_inputoutput.png) + +## 10. [Multi-Party Collaboration Authority Management](./fate_flow_authority_management.md) + +![](./images/fate_flow_authorization.png) \ No newline at end of file diff --git a/doc/fate_flow.zh.md b/doc/fate_flow.zh.md new file mode 100644 index 000000000..0aa3ec000 --- /dev/null +++ b/doc/fate_flow.zh.md @@ -0,0 +1,110 @@ +# 整体设计 + +## 1. 逻辑架构 + +- DSL定义作业 +- 自顶向下的纵向子任务流调度、多参与方联合子任务协调 +- 独立隔离的任务执行工作进程 +- 支持多类型多版本组件 +- 计算抽象API +- 存储抽象API +- 跨方传输抽象API + +![](./images/fate_flow_logical_arch.png) + +## 2. 整体架构 + +### 2.1 FATE整体架构 + +![](./images/fate_arch.png) + +### 2.2 FATE Flow整体架构 + +![](./images/fate_flow_arch.png) + +## 3. [调度架构](./fate_flow_job_scheduling.zh.md) + +### 3.1 基于共享状态的全新调度架构 + +- 剥离状态(资源、作业)与管理器(调度器、资源管理器) +- 资源状态与作业状态持久化存于MySQL，全局共享，提供可靠事务性操作 +- 提高管理服务的高可用与扩展性 +- 作业可介入，支持实现如重启、重跑、并行控制、资源隔离等 + +![](./images/fate_flow_scheduling_arch.png) + +### 3.2 状态驱动调度 + +- 资源协调 +- 拉起子进程Executor运行组件 +- Executor上报状态到本方Server，并且同时上报到调度方 +- 多方任务状态计算联邦任务状态 +- 上下游任务状态计算作业作态 + +![](./images/fate_flow_resource_process.png) + +## 4. [多方资源协调](./fate_flow_resource_management.zh.md) + +- 每个引擎总资源大小通过配置文件配置，后续实现系统对接 +- 总资源大小中的cores_per_node表示每个计算节点cpu核数，nodes表示计算节点个数 +- FATEFlow server启动时从配置文件读取资源大小配置，并注册更新到数据库 +- 以Job维度申请资源，Job Conf提交时生效，公式：task_parallelism*task_cores +- 详细请看文档单独章节 + +## 5. [数据流动追踪](./fate_flow_tracking.zh.md) + +- 定义 + - metric type: 指标类型，如auc, loss, ks等等 + - metric namespace: 自定义指标命名空间，如train, predict + - metric name: 自定义指标名称，如auc0，hetero_lr_auc0 + - metric data: key-value形式的指标数据 + - metric meta: key-value形式的指标元信息，支持灵活画图 +- API + - log_metric_data(metric_namespace, metric_name, metrics) + - set_metric_meta(metric_namespace, metric_name, metric_meta) + - get_metric_data(metric_namespace, metric_name) + - get_metric_meta(metric_namespace, metric_name) + +## 6. [作业实时监测](./fate_flow_monitoring.zh.md) + +- 工作进程存活性检测 +- 作业超时检测 +- 资源回收检测 +- 基础引擎会话超时检测 + +![](./images/fate_flow_detector.png) + +## 7. [任务组件中心](./fate_flow_component_registry.zh.md) + +![](./images/fate_flow_component_registry.png) + +## 8. [多方联合模型注册中心](./fate_flow_model_registry.zh.md) + +- 使用Google Protocol Buffer作为模型存储协议，利用跨语言共享，每个算法模型由两部分组成：ModelParam & ModelMeta +- 一个Pipeline产生一系列算法模型 +- 命名为Pipeline的模型存储Pipeline建模DSL及在线推理DSL +- 联邦学习下，需要保证所有参与方模型一致性，即模型绑定 +- model_key为用户提交任务时定义的模型标识 +- 联邦各方的模型ID由本方标识信息role、party_id，加model_key +- 联邦各方的模型版本必须唯一且保持一致，FATE-Flow直接设置为job_id + +![](./images/fate_flow_pipelined_model.png){: style="height:400px;width:450px"} + +![](./images/fate_flow_model_storage.png){: style="height:400px;width:800px"} + +## 9. [数据接入](./fate_flow_data_access.zh.md) + +- Upload： + - 外部存储直接导入到FATE Storage，创建一个新的DTable + - 作业运行时，Reader直接从Storage读取 + +- Table Bind： + - 外部存储地址关键到FATE一个新的DTable + - 作业运行时，Reader通过Meta从外部存储读取数据并转存到FATE Storage + - 打通大数据生态：HDFS，Hive/MySQL + +![](./images/fate_flow_inputoutput.png) + +## 10. [多方合作权限管理](./fate_flow_authority_management.zh.md) + +![](./images/fate_flow_authorization.png) diff --git a/doc/images/fate_arch.png b/doc/images/fate_arch.png new file mode 100644 index 000000000..bd8b2eda6 Binary files /dev/null and b/doc/images/fate_arch.png differ diff --git a/doc/images/fate_deploy_directory.png b/doc/images/fate_deploy_directory.png new file mode 100644 index 000000000..c255e05a1 Binary files /dev/null and b/doc/images/fate_deploy_directory.png differ diff --git a/doc/images/fate_flow_arch.png b/doc/images/fate_flow_arch.png new file mode 100644 index 000000000..2bb3e3e3d Binary files /dev/null and b/doc/images/fate_flow_arch.png differ diff --git a/doc/images/fate_flow_authorization.png b/doc/images/fate_flow_authorization.png new file mode 100644 index 000000000..8c6b5d18a Binary files /dev/null and b/doc/images/fate_flow_authorization.png differ diff --git a/doc/images/fate_flow_component_dsl.png b/doc/images/fate_flow_component_dsl.png new file mode 100644 index 000000000..24fc8064a Binary files /dev/null and b/doc/images/fate_flow_component_dsl.png differ diff --git a/doc/images/fate_flow_component_registry.png b/doc/images/fate_flow_component_registry.png new file mode 100644 index 000000000..ed2e8f780 Binary files /dev/null and b/doc/images/fate_flow_component_registry.png differ diff --git a/doc/images/fate_flow_dag.png b/doc/images/fate_flow_dag.png new file mode 100644 index 000000000..ca02a3ff5 Binary files /dev/null and b/doc/images/fate_flow_dag.png differ diff --git a/doc/images/fate_flow_detector.png b/doc/images/fate_flow_detector.png new file mode 100644 index 000000000..fb1dc62cc Binary files /dev/null and b/doc/images/fate_flow_detector.png differ diff --git a/doc/images/fate_flow_dsl.png b/doc/images/fate_flow_dsl.png new file mode 100644 index 000000000..cd9cd3373 Binary files /dev/null and b/doc/images/fate_flow_dsl.png differ diff --git a/doc/images/fate_flow_inputoutput.png b/doc/images/fate_flow_inputoutput.png new file mode 100644 index 000000000..2318d567e Binary files /dev/null and b/doc/images/fate_flow_inputoutput.png differ diff --git a/doc/images/fate_flow_logical_arch.png b/doc/images/fate_flow_logical_arch.png new file mode 100644 index 000000000..4c7677dac Binary files /dev/null and b/doc/images/fate_flow_logical_arch.png differ diff --git a/doc/images/fate_flow_major_feature.png b/doc/images/fate_flow_major_feature.png new file mode 100644 index 000000000..5d1eb71f6 Binary files /dev/null and b/doc/images/fate_flow_major_feature.png differ diff --git a/doc/images/fate_flow_model_storage.png b/doc/images/fate_flow_model_storage.png new file mode 100644 index 000000000..50b913e8d Binary files /dev/null and b/doc/images/fate_flow_model_storage.png differ diff --git a/doc/images/fate_flow_pipelined_model.png b/doc/images/fate_flow_pipelined_model.png new file mode 100644 index 000000000..ae20a0105 Binary files /dev/null and b/doc/images/fate_flow_pipelined_model.png differ diff --git a/doc/images/fate_flow_resource_process.png b/doc/images/fate_flow_resource_process.png new file mode 100644 index 000000000..b7275790c Binary files /dev/null and b/doc/images/fate_flow_resource_process.png differ diff --git a/doc/images/fate_flow_scheduling_arch.png b/doc/images/fate_flow_scheduling_arch.png new file mode 100644 index 000000000..77deb0368 Binary files /dev/null and b/doc/images/fate_flow_scheduling_arch.png differ diff --git a/doc/images/federated_learning_pipeline.png b/doc/images/federated_learning_pipeline.png new file mode 100644 index 000000000..97ad1e488 Binary files /dev/null and b/doc/images/federated_learning_pipeline.png differ diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 000000000..f0e348a2f --- /dev/null +++ b/doc/index.md @@ -0,0 +1,4 @@ +--- +template: overrides/home.html +title: Secure, Privacy-preserving Machine Learning Multi-Party Schduling System +--- diff --git a/doc/index.zh.md b/doc/index.zh.md new file mode 100644 index 000000000..31085da96 --- /dev/null +++ b/doc/index.zh.md @@ -0,0 +1,4 @@ +--- +template: overrides/home.zh.html +title: 安全，隐私保护的机器学习多方调度系统 +--- diff --git a/doc/mkdocs/README.md b/doc/mkdocs/README.md new file mode 100644 index 000000000..842eea2c2 --- /dev/null +++ b/doc/mkdocs/README.md @@ -0,0 +1,79 @@ +# Build + +## use docker + +At repo root, execute + +```sh +docker run --rm -it -p 8000:8000 -v ${PWD}:/docs sagewei0/mkdocs +``` + +to serve docs in http://localhost:8000 + +or + +```sh +docker run --rm -it -p 8000:8000 -v ${PWD}:/docs sagewei0/mkdocs build +``` + +to build docs to `site` folder. + +## manually + +[`mkdocs-material`](https://pypi.org/project/mkdocs-material/) and servel plugins are needed to build this docs + +Fisrt, create an python virtual environment + +```sh +python3 -m venv "fatedocs" +source fatedocs/bin/activate +pip install -U pip +``` +And then install requirements + +```sh +pip install -r doc/mkdocs/requirements.txt +``` + +Now, use + +```sh +mkdocs serve +``` + +at repo root to serve docs or + +use + +```sh +mkdocs build +``` + +at repo root to build docs to folder `site` + + +# Develop guide + +We use [mkdocs-material](https://squidfunk.github.io/mkdocs-material/) to build our docs. +Servel markdown extensions are really useful to write pretty documents such as +[admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/) and +[content-tabs](https://squidfunk.github.io/mkdocs-material/reference/content-tabs/). + +Servel plugins are introdused to makes mkdocs-material much powerful: + + +- [mkdocstrings](https://mkdocstrings.github.io/usage/) + automatic documentation from sources code. We mostly use this to automatic generate + `params api` for `federatedml`. + +- [awesome-pages](https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin) + for powerful nav rule + +- [i18n](https://ultrabug.github.io/mkdocs-static-i18n/) + for multi-languege support + +- [mkdocs-jupyter](https://github.com/danielfrg/mkdocs-jupyter) + for jupyter format support + +- [mkdocs-simple-hooks](https://github.com/aklajnert/mkdocs-simple-hooks) + for simple plugin-in \ No newline at end of file diff --git a/doc/mkdocs/assets/animations/ml.json b/doc/mkdocs/assets/animations/ml.json new file mode 100644 index 000000000..418556228 --- /dev/null +++ b/doc/mkdocs/assets/animations/ml.json @@ -0,0 +1 @@ +{"v":"5.5.7","meta":{"g":"LottieFiles AE 0.1.20","a":"","k":"","d":"","tc":""},"fr":60,"ip":0,"op":240,"w":180,"h":180,"nm":"Particles","ddd":0,"assets":[],"layers":[{"ddd":0,"ind":1,"ty":4,"nm":"Yellow Ball","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[155.5,94.5,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[131.5,94.5,0],"to":[0,0,0],"ti":[0,0,0]},{"t":240,"s":[155.5,94.5,0]}],"ix":2},"a":{"a":0,"k":[14,-51.5,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"d":1,"ty":"el","s":{"a":0,"k":[32,32],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"nm":"Ellipse Path 1","mn":"ADBE Vector Shape - Ellipse","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.964705942191,0.85882358925,0.250980392157,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[15,-51.5],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Ellipse 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":2,"ty":4,"nm":"Yellow Ball 2","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[39.5,134.5,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[39.5,110.5,0],"to":[0,0,0],"ti":[0,0,0]},{"t":240,"s":[39.5,134.5,0]}],"ix":2},"a":{"a":0,"k":[14,-51.5,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"d":1,"ty":"el","s":{"a":0,"k":[32,32],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"nm":"Ellipse Path 1","mn":"ADBE Vector Shape - Ellipse","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.964705942191,0.85882358925,0.250980392157,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[15,-51.5],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Ellipse 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":3,"ty":4,"nm":"Red Ball","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[109.97,77.248,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[82.22,101.248,0],"to":[0,0,0],"ti":[0,0,0]},{"t":240,"s":[109.97,77.248,0]}],"ix":2},"a":{"a":0,"k":[14,-51.5,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"d":1,"ty":"el","s":{"a":0,"k":[32,32],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"nm":"Ellipse Path 1","mn":"ADBE Vector Shape - Ellipse","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.85882358925,0.325490196078,0.356862745098,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[15,-51.5],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Ellipse 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":4,"ty":4,"nm":"Blue Ball","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[68.75,95.25,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[27.5,60.5,0],"to":[0,0,0],"ti":[0,0,0]},{"t":240,"s":[68.75,95.25,0]}],"ix":2},"a":{"a":0,"k":[14,-51.5,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"d":1,"ty":"el","s":{"a":0,"k":[32,32],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"nm":"Ellipse Path 1","mn":"ADBE Vector Shape - Ellipse","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.396078461292,0.6,0.929411824544,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[15,-51.5],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Ellipse 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":5,"ty":4,"nm":"Light Blue Ball","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[79.5,145.5,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[103.5,145.5,0],"to":[0,0,0],"ti":[0,0,0]},{"t":240,"s":[79.5,145.5,0]}],"ix":2},"a":{"a":0,"k":[14,-51.5,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"d":1,"ty":"el","s":{"a":0,"k":[32,32],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"nm":"Ellipse Path 1","mn":"ADBE Vector Shape - Ellipse","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.443137284821,0.823529471603,0.87450986376,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[15,-51.5],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Ellipse 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":6,"ty":4,"nm":"Violet Ball","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[87.5,43.5,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[103.5,42.5,0],"to":[0,0,0],"ti":[0,0,0]},{"t":240,"s":[87.5,43.5,0]}],"ix":2},"a":{"a":0,"k":[14,-51.5,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"d":1,"ty":"el","s":{"a":0,"k":[32,32],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"nm":"Ellipse Path 1","mn":"ADBE Vector Shape - Ellipse","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.513725490196,0.376470618154,0.956862804936,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[15,-51.5],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Ellipse 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":7,"ty":4,"nm":"Line 1","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[66.375,53.25,0],"ix":2},"a":{"a":0,"k":[-23.625,-36.75,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-20.25,5.75],[-1.25,-46.75]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-61.25,-29.25],[15,-47.5]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-20.25,5.75],[-1.25,-46.75]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":8,"ty":4,"nm":"Line 2","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[119.25,68.75,0],"ix":2},"a":{"a":0,"k":[29.25,-21.25,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-1.75,-46.5],[66.75,5]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[14.5,-47.5],[42.5,5]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-1.75,-46.5],[66.75,5]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":9,"ty":4,"nm":"Line 3","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[118.125,119.75,0],"ix":2},"a":{"a":0,"k":[28.125,29.75,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[66.5,4.75],[-9.75,56.5]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[42.5,4.75],[15,56]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[66.5,4.75],[-9.75,56.5]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":10,"ty":4,"nm":"Line 4","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[66,133.5,0],"ix":2},"a":{"a":0,"k":[-24,43.5,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-49.5,44.25],[-9.75,56.5]],"c":true}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-49.75,20.5],[14.25,56.5]],"c":true}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-49.5,44.25],[-9.75,56.5]],"c":true}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":11,"ty":4,"nm":"Line 5","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[68,82.625,0],"ix":2},"a":{"a":0,"k":[-22,-7.375,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-49.25,45],[-1.75,-46.5]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-49.5,20.75],[14.25,-48]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-49.25,45],[-1.75,-46.5]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":12,"ty":4,"nm":"Line 6","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[65.25,104.875,0],"ix":2},"a":{"a":0,"k":[-24.75,14.875,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-20,5],[-9.75,56]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-61.75,-29.75],[14.5,56]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-20,5],[-9.75,56]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":13,"ty":4,"nm":"Line 7","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[92.75,118.875,0],"ix":2},"a":{"a":0,"k":[2.75,28.875,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[21,-12.25],[-9.75,56.25]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-7,11.5],[14.5,56]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[21,-12.25],[-9.75,56.25]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":14,"ty":4,"nm":"Line 8","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[65.5,77.625,0],"ix":2},"a":{"a":0,"k":[-24.5,-12.375,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-20.75,5.5],[20.5,-12.5]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-62,-29.5],[-7.25,11.25]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-20.75,5.5],[20.5,-12.5]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0},{"ddd":0,"ind":15,"ty":4,"nm":"Line 9","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[118.375,93.375,0],"ix":2},"a":{"a":0,"k":[28.375,3.375,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":0,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[20.375,-12.625],[67.125,5.5]],"c":false}]},{"i":{"x":0.22,"y":1},"o":{"x":0.78,"y":0},"t":120,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[-7,11.5],[42.5,4.75]],"c":false}]},{"t":240,"s":[{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[20.375,-12.625],[67.125,5.5]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.109803929048,0.098039223166,0.101960791794,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":5,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":241,"st":0,"bm":0}],"markers":[]} \ No newline at end of file diff --git a/doc/mkdocs/css/custom.css b/doc/mkdocs/css/custom.css new file mode 100644 index 000000000..f1b9f54bf --- /dev/null +++ b/doc/mkdocs/css/custom.css @@ -0,0 +1,39 @@ +/* Indentation. */ +div.doc-contents:not(.first) { + padding-left: 25px; + border-left: 4px solid rgba(230, 230, 230); + margin-bottom: 80px; +} + +/* Don't capitalize names. */ +h5.doc-heading { + text-transform: none !important; +} + +/* Don't use vertical space on hidden ToC entries. */ +h6.hidden-toc { + margin: 0 !important; + position: relative; + top: -70px; +} + +h6.hidden-toc::before { + margin-top: 0 !important; + padding-top: 0 !important; +} + +/* Don't show permalink of hidden ToC entries. */ +h6.hidden-toc a.headerlink { + display: none; +} + +/* Avoid breaking parameters name, etc. in table cells. */ +td code { + word-break: normal !important; +} + +/* For pieces of Markdown rendered in table cells. */ +td p { + margin-top: 0 !important; + margin-bottom: 0 !important; +} diff --git a/doc/mkdocs/css/extra.css b/doc/mkdocs/css/extra.css new file mode 100644 index 000000000..ede6e6139 --- /dev/null +++ b/doc/mkdocs/css/extra.css @@ -0,0 +1,34 @@ +/* Remove default title on the page */ +.md-content__inner h1:first-child { + display: none; +} + +/* Adjust to 2px to align with the title */ +.md-logo { + padding-top: 6px; +} + +.btn { + border: none; + padding: 14px 28px; + cursor: pointer; + display: inline-block; + + background: #009688; + color: white; +} + +.btn:hover { + background: #00bfa5; + color: white; +} + +.center { + display: block; + margin-left: auto; + margin-right: auto; +} + +.text-center { + text-align: center; +} diff --git a/doc/mkdocs/css/landing.css b/doc/mkdocs/css/landing.css new file mode 100644 index 000000000..e0ebdb147 --- /dev/null +++ b/doc/mkdocs/css/landing.css @@ -0,0 +1,144 @@ +.tx-container { + background: + linear-gradient(to bottom, var(--md-primary-fg-color), #2196f3 100%, var(--md-default-bg-color) 100%) +} + +[data-md-color-scheme=slate] .tx-container { + background: + linear-gradient(to bottom, var(--md-primary-fg-color), #2196f3 100%, var(--md-default-bg-color) 100%) +} + +.tx-landing { + margin: 0 .8rem; + color: var(--md-primary-bg-color) +} + +.tx-landing h1 { + margin-bottom: 1rem; + color: currentColor; + font-weight: 700 +} + +@media screen and (max-width: 30em) { + .tx-landing h1 { + font-size: 1.4rem + } +} + +.tx-landing__content p a { + color: inherit; + text-decoration: underline; +} + +.tx-landing__testimonials { + width: 100%; + text-align: center; +} + +.tx-landing__content p a:hover { + color: darkblue; + text-decoration: underline; +} + +.tx-landing__logos { + display: flex; + flex-direction: row; + flex-wrap: wrap; + justify-content: center; +} + +.tx-landing__logos img { + height: 8vh; + max-height: 81px; /* max height of images */ + width: auto; + margin: 2vh; + vertical-align: middle; +} + +.tx-landing__quotes { + padding-bottom: 5em; + text-align: center; +} + +@media screen and (min-width: 60em) { + .tx-landing__quotes { + margin: 1em 5em; + } +} + +.tx-landing__quotes figure { + margin: 2em auto 2em auto; +} + +.tx-landing__quote { + display: flex; + border-radius: 1em; + padding: 1em 1em 0 1em; + background: var(--md-primary-fg-color); +} + +.tx-landing__quote blockquote { + border: 0; + color: #fff; +} + +.tx-landing__quote a img { + height: 6vh; + max-height: 81px; /* max height of images */ + display: block; + margin-left: auto; + margin-right: auto; +} + +@media screen and (min-width: 60em) { + .tx-container { + padding-bottom: 14vw + } + + .tx-landing { + display: flex; + align-items: stretch + } + + .tx-landing__content { + max-width: 24rem; + margin-top: 3.5rem; + } + + .tx-landing__image { + order: 1; + width: 38rem; + transform: translateX(4rem) + } +} + +@media screen and (min-width: 77em) { + .tx-landing__image { + transform: translateX(8rem) + } +} + +.tx-landing .md-button { + margin-top: .5rem; + margin-right: .5rem; + color: var(--md-primary-bg-color) +} + +.tx-landing .md-button:hover, .tx-landing .md-button:focus { + color: var(--md-default-bg-color); + background-color: #8bc34a; + border-color: #8bc34a +} + +.md-typeset lottie-player { + max-width: 100%; + height: auto; +} + +.md-announce a { + color: var(--md-primary-bg-color); +} + +.md-banner a { + color: var(--md-primary-bg-color); +} diff --git a/doc/mkdocs/css/termynal.css b/doc/mkdocs/css/termynal.css new file mode 100644 index 000000000..f02626e3d --- /dev/null +++ b/doc/mkdocs/css/termynal.css @@ -0,0 +1,110 @@ +/** + * termynal.js + * + * @author Ines Montani + * @version 0.0.1 + * @license MIT + */ + +:root { + --color-bg: #eee8d5; + --color-text: #073642; + --color-text-subtle: #cb4b16; + background: linear-gradient(to right, #3a1c71, #d76d77, #ffaf7b); +} + +[data-termynal] { + width: 750px; + max-width: 100%; + background: var(--color-bg); + color: var(--color-text); + font-size: 18px; + /* font-family: 'Fira Mono', Consolas, Menlo, Monaco, 'Courier New', Courier, monospace; */ + font-family: 'Roboto Mono', 'Fira Mono', Consolas, Menlo, Monaco, 'Courier New', Courier, monospace; + border-radius: 4px; + padding: 75px 45px 35px; + position: relative; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +[data-termynal]:before { + content: ''; + position: absolute; + top: 15px; + left: 15px; + display: inline-block; + width: 15px; + height: 15px; + border-radius: 50%; + /* A little hack to display the window buttons in one pseudo element. */ + background: #d9515d; + -webkit-box-shadow: 25px 0 0 #f4c025, 50px 0 0 #3ec930; + box-shadow: 25px 0 0 #f4c025, 50px 0 0 #3ec930; +} + +[data-termynal]:after { + content: 'bash'; + position: absolute; + color: var(--color-text-subtle); + top: 5px; + left: 0; + width: 100%; + text-align: center; +} + +a[data-terminal-control] { + text-align: right; + display: block; + color: #aebbff; +} + +[data-ty] { + display: block; + line-height: 2; +} + +[data-ty]:before { + /* Set up defaults and ensure empty lines are displayed. */ + content: ''; + display: inline-block; + vertical-align: middle; +} + +[data-ty="input"]:before, +[data-ty-prompt]:before { + margin-right: 0.75em; + color: var(--color-text-subtle); +} + +[data-ty="input"]:before { + content: '$'; +} + +[data-ty][data-ty-prompt]:before { + content: attr(data-ty-prompt); +} + +[data-ty-cursor]:after { + content: attr(data-ty-cursor); + font-family: monospace; + margin-left: 0.5em; + -webkit-animation: blink 1s infinite; + animation: blink 1s infinite; +} + + +/* Cursor animation */ + +@-webkit-keyframes blink { + 50% { + opacity: 0; + } +} + +@keyframes blink { + 50% { + opacity: 0; + } +} + diff --git a/doc/mkdocs/docker/Dockerfile b/doc/mkdocs/docker/Dockerfile new file mode 100644 index 000000000..1b48139d7 --- /dev/null +++ b/doc/mkdocs/docker/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.9.2-alpine3.13 + +# Environment variables +ENV PACKAGES=/usr/local/lib/python3.9/site-packages +ENV PYTHONDONTWRITEBYTECODE=1 + +# Set build directory +WORKDIR /tmp + +COPY requirements.txt . + +RUN set -e ;\ + apk upgrade --update-cache -a ;\ + apk add --no-cache libstdc++ libffi-dev ;\ + apk add --no-cache --virtual .build gcc g++ musl-dev python3-dev cargo openssl-dev git;\ + pip install --no-cache-dir -r requirements.txt + +# clean +RUN apk del .build ;\ + rm -rf /tmp/* /root/.cache + +# Set working directory +WORKDIR /docs + +# Expose MkDocs development server port +EXPOSE 8000 + +ENV PYTHONPATH=$PYTHONPATH:/docs/python +# Start development server by default +ENTRYPOINT ["mkdocs"] +CMD ["serve", "--dev-addr=0.0.0.0:8000"] \ No newline at end of file diff --git a/doc/mkdocs/docker/README.md b/doc/mkdocs/docker/README.md new file mode 100644 index 000000000..5c5466b0a --- /dev/null +++ b/doc/mkdocs/docker/README.md @@ -0,0 +1,25 @@ +# Image for build FATE's documents + +This image is modified from [mkdocs-meterial](https://squidfunk.github.io/mkdocs-material/) with some plugins embeded. + +Usage + +Mount the folder where your mkdocs.yml resides as a volume into /docs: + +- Start development server on http://localhost:8000 + +```console +docker run --rm -it -p 8000:8000 -v ${PWD}:/docs sagewei0/mkdocs +``` + +- Build documentation + +```console +docker run --rm -it -v ${PWD}:/docs sagewei/mkdocs build +``` + +- Deploy documentation to GitHub Pages + +```console +docker run --rm -it -v ~/.ssh:/root/.ssh -v ${PWD}:/docs sagewei0/mkdocs gh-deploy +``` diff --git a/doc/mkdocs/js/custom.js b/doc/mkdocs/js/custom.js new file mode 100644 index 000000000..a0e16abb7 --- /dev/null +++ b/doc/mkdocs/js/custom.js @@ -0,0 +1,106 @@ +document.querySelectorAll(".use-termynal").forEach(node => { + node.style.display = "block"; + new Termynal(node, { + lineDelay: 500 + }); +}); +const progressLiteralStart = "---> 100%"; +const promptLiteralStart = "$ "; +const customPromptLiteralStart = "# "; +const termynalActivateClass = "termy"; +let termynals = []; + +function createTermynals() { + document + .querySelectorAll(`.${termynalActivateClass} .highlight`) + .forEach(node => { + const text = node.textContent; + const lines = text.split("\n"); + const useLines = []; + let buffer = []; + function saveBuffer() { + if (buffer.length) { + let isBlankSpace = true; + buffer.forEach(line => { + if (line) { + isBlankSpace = false; + } + }); + dataValue = {}; + if (isBlankSpace) { + dataValue["delay"] = 0; + } + if (buffer[buffer.length - 1] === "") { + // A last single
won't have effect + // so put an additional one + buffer.push(""); + } + const bufferValue = buffer.join("
"); + dataValue["value"] = bufferValue; + useLines.push(dataValue); + buffer = []; + } + } + for (let line of lines) { + if (line === progressLiteralStart) { + saveBuffer(); + useLines.push({ + type: "progress" + }); + } else if (line.startsWith(promptLiteralStart)) { + saveBuffer(); + const value = line.replace(promptLiteralStart, "").trimEnd(); + useLines.push({ + type: "input", + value: value + }); + } else if (line.startsWith("// ")) { + saveBuffer(); + const value = "💬 " + line.replace("// ", "").trimEnd(); + useLines.push({ + value: value, + class: "termynal-comment", + delay: 0 + }); + } else if (line.startsWith(customPromptLiteralStart)) { + saveBuffer(); + const promptStart = line.indexOf(promptLiteralStart); + if (promptStart === -1) { + console.error("Custom prompt found but no end delimiter", line) + } + const prompt = line.slice(0, promptStart).replace(customPromptLiteralStart, "") + let value = line.slice(promptStart + promptLiteralStart.length); + useLines.push({ + type: "input", + value: value, + prompt: prompt + }); + } else { + buffer.push(line); + } + } + saveBuffer(); + const div = document.createElement("div"); + node.replaceWith(div); + const termynal = new Termynal(div, { + lineData: useLines, + noInit: true, + lineDelay: 500 + }); + termynals.push(termynal); + }); +} + +function loadVisibleTermynals() { + termynals = termynals.filter(termynal => { + if (termynal.container.getBoundingClientRect().top - innerHeight <= 0) { + termynal.init(); + return false; + } + return true; + }); +} +window.addEventListener("scroll", loadVisibleTermynals); +createTermynals(); +loadVisibleTermynals(); + diff --git a/doc/mkdocs/js/lottie-player.js b/doc/mkdocs/js/lottie-player.js new file mode 100644 index 000000000..938688078 --- /dev/null +++ b/doc/mkdocs/js/lottie-player.js @@ -0,0 +1,181 @@ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e((t="undefined"!=typeof globalThis?globalThis:t||self)["lottie-player"]={})}(this,(function(exports){"use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}var REACT_ELEMENT_TYPE;function _jsx(t,e,r,i){REACT_ELEMENT_TYPE||(REACT_ELEMENT_TYPE="function"==typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103);var a=t&&t.defaultProps,s=arguments.length-3;if(e||0===s||(e={children:void 0}),1===s)e.children=i;else if(s>1){for(var n=new Array(s),o=0;o=0||(a[r]=t[r]);return a}function _objectWithoutProperties(t,e){if(null==t)return{};var r,i,a=_objectWithoutPropertiesLoose(t,e);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(t);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(t,r)&&(a[r]=t[r])}return a}function _assertThisInitialized(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}function _possibleConstructorReturn(t,e){return!e||"object"!=typeof e&&"function"!=typeof e?_assertThisInitialized(t):e}function _createSuper(t){var e=_isNativeReflectConstruct();return function(){var r,i=_getPrototypeOf(t);if(e){var a=_getPrototypeOf(this).constructor;r=Reflect.construct(i,arguments,a)}else r=i.apply(this,arguments);return _possibleConstructorReturn(this,r)}}function _superPropBase(t,e){for(;!Object.prototype.hasOwnProperty.call(t,e)&&null!==(t=_getPrototypeOf(t)););return t}function _get(t,e,r){return(_get="undefined"!=typeof Reflect&&Reflect.get?Reflect.get:function(t,e,r){var i=_superPropBase(t,e);if(i){var a=Object.getOwnPropertyDescriptor(i,e);return a.get?a.get.call(r):a.value}})(t,e,r||t)}function set(t,e,r,i){return(set="undefined"!=typeof Reflect&&Reflect.set?Reflect.set:function(t,e,r,i){var a,s=_superPropBase(t,e);if(s){if((a=Object.getOwnPropertyDescriptor(s,e)).set)return a.set.call(i,r),!0;if(!a.writable)return!1}if(a=Object.getOwnPropertyDescriptor(i,e)){if(!a.writable)return!1;a.value=r,Object.defineProperty(i,e,a)}else _defineProperty(i,e,r);return!0})(t,e,r,i)}function _set(t,e,r,i,a){if(!set(t,e,r,i||t)&&a)throw new Error("failed to set property");return r}function _taggedTemplateLiteral(t,e){return e||(e=t.slice(0)),Object.freeze(Object.defineProperties(t,{raw:{value:Object.freeze(e)}}))}function _taggedTemplateLiteralLoose(t,e){return e||(e=t.slice(0)),t.raw=e,t}function _readOnlyError(t){throw new TypeError('"'+t+'" is read-only')}function _writeOnlyError(t){throw new TypeError('"'+t+'" is write-only')}function _classNameTDZError(t){throw new Error('Class "'+t+'" cannot be referenced in computed property keys.')}function _temporalUndefined(){}function _tdz(t){throw new ReferenceError(t+" is not defined - temporal dead zone")}function _temporalRef(t,e){return t===_temporalUndefined?_tdz(e):t}function _slicedToArray(t,e){return _arrayWithHoles(t)||_iterableToArrayLimit(t,e)||_unsupportedIterableToArray(t,e)||_nonIterableRest()}function _slicedToArrayLoose(t,e){return _arrayWithHoles(t)||_iterableToArrayLimitLoose(t,e)||_unsupportedIterableToArray(t,e)||_nonIterableRest()}function _toArray(t){return _arrayWithHoles(t)||_iterableToArray(t)||_unsupportedIterableToArray(t)||_nonIterableRest()}function _toConsumableArray(t){return _arrayWithoutHoles(t)||_iterableToArray(t)||_unsupportedIterableToArray(t)||_nonIterableSpread()}function _arrayWithoutHoles(t){if(Array.isArray(t))return _arrayLikeToArray(t)}function _arrayWithHoles(t){if(Array.isArray(t))return t}function _maybeArrayLike(t,e,r){if(e&&!Array.isArray(e)&&"number"==typeof e.length){var i=e.length;return _arrayLikeToArray(e,void 0!==r&&rt.length)&&(e=t.length);for(var r=0,i=new Array(e);r=t.length?{done:!0}:{done:!1,value:t[i++]}},e:function(t){throw t},f:a}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var s,n=!0,o=!1;return{s:function(){r=t[Symbol.iterator]()},n:function(){var t=r.next();return n=t.done,t},e:function(t){o=!0,s=t},f:function(){try{n||null==r.return||r.return()}finally{if(o)throw s}}}}function _createForOfIteratorHelperLoose(t,e){var r;if("undefined"==typeof Symbol||null==t[Symbol.iterator]){if(Array.isArray(t)||(r=_unsupportedIterableToArray(t))||e&&t&&"number"==typeof t.length){r&&(t=r);var i=0;return function(){return i>=t.length?{done:!0}:{done:!1,value:t[i++]}}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}return(r=t[Symbol.iterator]()).next.bind(r)}function _skipFirstGeneratorNext(t){return function(){var e=t.apply(this,arguments);return e.next(),e}}function _toPrimitive(t,e){if("object"!=typeof t||null===t)return t;var r=t[Symbol.toPrimitive];if(void 0!==r){var i=r.call(t,e||"default");if("object"!=typeof i)return i;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===e?String:Number)(t)}function _toPropertyKey(t){var e=_toPrimitive(t,"string");return"symbol"==typeof e?e:String(e)}function _initializerWarningHelper(t,e){throw new Error("Decorating class property failed. Please ensure that proposal-class-properties is enabled and runs after the decorators transform.")}function _initializerDefineProperty(t,e,r,i){r&&Object.defineProperty(t,e,{enumerable:r.enumerable,configurable:r.configurable,writable:r.writable,value:r.initializer?r.initializer.call(i):void 0})}function _applyDecoratedDescriptor(t,e,r,i,a){var s={};return Object.keys(i).forEach((function(t){s[t]=i[t]})),s.enumerable=!!s.enumerable,s.configurable=!!s.configurable,("value"in s||s.initializer)&&(s.writable=!0),s=r.slice().reverse().reduce((function(r,i){return i(t,e,r)||r}),s),a&&void 0!==s.initializer&&(s.value=s.initializer?s.initializer.call(a):void 0,s.initializer=void 0),void 0===s.initializer&&(Object.defineProperty(t,e,s),s=null),s}"function"==typeof Symbol&&Symbol.asyncIterator&&(_AsyncGenerator.prototype[Symbol.asyncIterator]=function(){return this}),_AsyncGenerator.prototype.next=function(t){return this._invoke("next",t)},_AsyncGenerator.prototype.throw=function(t){return this._invoke("throw",t)},_AsyncGenerator.prototype.return=function(t){return this._invoke("return",t)};var id=0;function _classPrivateFieldLooseKey(t){return"__private_"+id+++"_"+t}function _classPrivateFieldLooseBase(t,e){if(!Object.prototype.hasOwnProperty.call(t,e))throw new TypeError("attempted to use private field on non-instance");return t}function _classPrivateFieldGet(t,e){return _classApplyDescriptorGet(t,_classExtractFieldDescriptor(t,e,"get"))}function _classPrivateFieldSet(t,e,r){return _classApplyDescriptorSet(t,_classExtractFieldDescriptor(t,e,"set"),r),r}function _classPrivateFieldDestructureSet(t,e){return _classApplyDescriptorDestructureSet(t,_classExtractFieldDescriptor(t,e,"set"))}function _classExtractFieldDescriptor(t,e,r){if(!e.has(t))throw new TypeError("attempted to "+r+" private field on non-instance");return e.get(t)}function _classStaticPrivateFieldSpecGet(t,e,r){return _classCheckPrivateStaticAccess(t,e),_classCheckPrivateStaticFieldDescriptor(r,"get"),_classApplyDescriptorGet(t,r)}function _classStaticPrivateFieldSpecSet(t,e,r,i){return _classCheckPrivateStaticAccess(t,e),_classCheckPrivateStaticFieldDescriptor(r,"set"),_classApplyDescriptorSet(t,r,i),i}function _classStaticPrivateMethodGet(t,e,r){return _classCheckPrivateStaticAccess(t,e),r}function _classStaticPrivateMethodSet(){throw new TypeError("attempted to set read only static private field")}function _classApplyDescriptorGet(t,e){return e.get?e.get.call(t):e.value}function _classApplyDescriptorSet(t,e,r){if(e.set)e.set.call(t,r);else{if(!e.writable)throw new TypeError("attempted to set read only private field");e.value=r}}function _classApplyDescriptorDestructureSet(t,e){if(e.set)return"__destrObj"in e||(e.__destrObj={set value(r){e.set.call(t,r)}}),e.__destrObj;if(!e.writable)throw new TypeError("attempted to set read only private field");return e}function _classStaticPrivateFieldDestructureSet(t,e,r){return _classCheckPrivateStaticAccess(t,e),_classCheckPrivateStaticFieldDescriptor(r,"set"),_classApplyDescriptorDestructureSet(t,r)}function _classCheckPrivateStaticAccess(t,e){if(t!==e)throw new TypeError("Private static access of wrong provenance")}function _classCheckPrivateStaticFieldDescriptor(t,e){if(void 0===t)throw new TypeError("attempted to "+e+" private static field before its declaration")}function _decorate(t,e,r,i){var a=_getDecoratorsApi();if(i)for(var s=0;s=0;s--){var n=e[t.placement];n.splice(n.indexOf(t.key),1);var o=this.fromElementDescriptor(t),h=this.toElementFinisherExtras((0,a[s])(o)||o);t=h.element,this.addElementPlacement(t,e),h.finisher&&i.push(h.finisher);var l=h.extras;if(l){for(var p=0;p=0;i--){var a=this.fromClassDescriptor(t),s=this.toClassDescriptor((0,e[i])(a)||a);if(void 0!==s.finisher&&r.push(s.finisher),void 0!==s.elements){t=s.elements;for(var n=0;n]+)>/g,(function(t,e){return"$"+r[e]})))}if("function"==typeof e){var s=this;return i[Symbol.replace].call(this,t,(function(){var t=[];return t.push.apply(t,arguments),"object"!=typeof t[t.length-1]&&t.push(n(t,s)),e.apply(this,t)}))}return i[Symbol.replace].call(this,t,e)},_wrapRegExp.apply(this,arguments)} +/*! ***************************************************************************** + Copyright (c) Microsoft Corporation. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. + ***************************************************************************** */var _extendStatics=function(t,e){return(_extendStatics=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&(t[r]=e[r])})(t,e)};function __extends(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Class extends value "+String(e)+" is not a constructor or null");function r(){this.constructor=t}_extendStatics(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}var _assign=function(){return(_assign=Object.assign||function(t){for(var e,r=1,i=arguments.length;r=0;o--)(a=t[o])&&(n=(s<3?a(n):s>3?a(e,r,n):a(e,r))||n);return s>3&&n&&Object.defineProperty(e,r,n),n}function __param(t,e){return function(r,i){e(r,i,t)}}function __metadata(t,e){if("object"==typeof Reflect&&"function"==typeof Reflect.metadata)return Reflect.metadata(t,e)}function __awaiter(t,e,r,i){return new(r||(r=Promise))((function(a,s){function n(t){try{h(i.next(t))}catch(t){s(t)}}function o(t){try{h(i.throw(t))}catch(t){s(t)}}function h(t){var e;t.done?a(t.value):(e=t.value,e instanceof r?e:new r((function(t){t(e)}))).then(n,o)}h((i=i.apply(t,e||[])).next())}))}function __generator(t,e){var r,i,a,s,n={label:0,sent:function(){if(1&a[0])throw a[1];return a[1]},trys:[],ops:[]};return s={next:o(0),throw:o(1),return:o(2)},"function"==typeof Symbol&&(s[Symbol.iterator]=function(){return this}),s;function o(s){return function(o){return function(s){if(r)throw new TypeError("Generator is already executing.");for(;n;)try{if(r=1,i&&(a=2&s[0]?i.return:s[0]?i.throw||((a=i.return)&&a.call(i),0):i.next)&&!(a=a.call(i,s[1])).done)return a;switch(i=0,a&&(s=[2&s[0],a.value]),s[0]){case 0:case 1:a=s;break;case 4:return n.label++,{value:s[1],done:!1};case 5:n.label++,i=s[1],s=[0];continue;case 7:s=n.ops.pop(),n.trys.pop();continue;default:if(!(a=n.trys,(a=a.length>0&&a[a.length-1])||6!==s[0]&&2!==s[0])){n=0;continue}if(3===s[0]&&(!a||s[1]>a[0]&&s[1]=t.length&&(t=void 0),{value:t&&t[i++],done:!t}}};throw new TypeError(e?"Object is not iterable.":"Symbol.iterator is not defined.")}function __read(t,e){var r="function"==typeof Symbol&&t[Symbol.iterator];if(!r)return t;var i,a,s=r.call(t),n=[];try{for(;(void 0===e||e-- >0)&&!(i=s.next()).done;)n.push(i.value)}catch(t){a={error:t}}finally{try{i&&!i.done&&(r=s.return)&&r.call(s)}finally{if(a)throw a.error}}return n}function __spread(){for(var t=[],e=0;e1||o(t,e)}))})}function o(t,e){try{(r=a[t](e)).value instanceof __await?Promise.resolve(r.value.v).then(h,l):p(s[0][2],r)}catch(t){p(s[0][3],t)}var r}function h(t){o("next",t)}function l(t){o("throw",t)}function p(t,e){t(e),s.shift(),s.length&&o(s[0][0],s[0][1])}}function __asyncDelegator(t){var e,r;return e={},i("next"),i("throw",(function(t){throw t})),i("return"),e[Symbol.iterator]=function(){return this},e;function i(i,a){e[i]=t[i]?function(e){return(r=!r)?{value:__await(t[i](e)),done:"return"===i}:a?a(e):e}:a}}function __asyncValues(t){if(!Symbol.asyncIterator)throw new TypeError("Symbol.asyncIterator is not defined.");var e,r=t[Symbol.asyncIterator];return r?r.call(t):(t="function"==typeof __values?__values(t):t[Symbol.iterator](),e={},i("next"),i("throw"),i("return"),e[Symbol.asyncIterator]=function(){return this},e);function i(r){e[r]=t[r]&&function(e){return new Promise((function(i,a){(function(t,e,r,i){Promise.resolve(i).then((function(e){t({value:e,done:r})}),e)})(i,a,(e=t[r](e)).done,e.value)}))}}}function __makeTemplateObject(t,e){return Object.defineProperty?Object.defineProperty(t,"raw",{value:e}):t.raw=e,t}var __setModuleDefault=Object.create?function(t,e){Object.defineProperty(t,"default",{enumerable:!0,value:e})}:function(t,e){t.default=e};function __importStar(t){if(t&&t.__esModule)return t;var e={};if(null!=t)for(var r in t)"default"!==r&&Object.prototype.hasOwnProperty.call(t,r)&&__createBinding(e,t,r);return __setModuleDefault(e,t),e}function __importDefault(t){return t&&t.__esModule?t:{default:t}}function __classPrivateFieldGet(t,e,r,i){if("a"===r&&!i)throw new TypeError("Private accessor was defined without a getter");if("function"==typeof e?t!==e||!i:!e.has(t))throw new TypeError("Cannot read private member from an object whose class did not declare it");return"m"===r?i:"a"===r?i.call(t):i?i.value:e.get(t)}function __classPrivateFieldSet(t,e,r,i,a){if("m"===i)throw new TypeError("Private method is not writable");if("a"===i&&!a)throw new TypeError("Private accessor was defined without a setter");if("function"==typeof e?t!==e||!a:!e.has(t))throw new TypeError("Cannot write private member to an object whose class did not declare it");return"a"===i?a.call(t,r):a?a.value=r:e.set(t,r),r +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */}var isCEPolyfill="undefined"!=typeof window&&null!=window.customElements&&void 0!==window.customElements.polyfillWrapFlushCallback,reparentNodes=function(t,e){for(var r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null,i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:null;e!==r;){var a=e.nextSibling;t.insertBefore(e,i),e=a}},removeNodes=function(t,e){for(var r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null;e!==r;){var i=e.nextSibling;t.removeChild(e),e=i}},marker="{{lit-".concat(String(Math.random()).slice(2),"}}"),nodeMarker="\x3c!--".concat(marker,"--\x3e"),markerRegex=new RegExp("".concat(marker,"|").concat(nodeMarker)),boundAttributeSuffix="$lit$";class Template{constructor(t,e){this.parts=[],this.element=e;for(var r=[],i=[],a=document.createTreeWalker(e.content,133,null,!1),s=0,n=-1,o=0,{strings:h,values:{length:l}}=t;o0;){var u=h[o],y=lastAttributeNameRegex.exec(u)[2],g=y.toLowerCase()+boundAttributeSuffix,v=p.getAttribute(g);p.removeAttribute(g);var b=v.split(markerRegex);this.parts.push({type:"attribute",index:n,name:y,strings:b}),o+=b.length-1}}"TEMPLATE"===p.tagName&&(i.push(p),a.currentNode=p.content)}else if(3===p.nodeType){var _=p.data;if(_.indexOf(marker)>=0){for(var P=p.parentNode,S=_.split(markerRegex),E=S.length-1,x=0;x{var r=t.length-e.length;return r>=0&&t.slice(r)===e},isTemplatePartActive=t=>-1!==t.index,createMarker=()=>document.createComment(""),lastAttributeNameRegex=/([ \x09\x0a\x0c\x0d])([^\0-\x1F\x7F-\x9F "'>=/]+)([ \x09\x0a\x0c\x0d]*=[ \x09\x0a\x0c\x0d]*(?:[^ \x09\x0a\x0c\x0d"'`<>=]*|"[^"]*|'[^']*))$/,walkerNodeFilter=133;function removeNodesFromTemplate(t,e){for(var{element:{content:r},parts:i}=t,a=document.createTreeWalker(r,walkerNodeFilter,null,!1),s=nextActiveIndexInTemplateParts(i),n=i[s],o=-1,h=0,l=[],p=null;a.nextNode();){o++;var c=a.currentNode;for(c.previousSibling===p&&(p=null),e.has(c)&&(l.push(c),null===p&&(p=c)),null!==p&&h++;void 0!==n&&n.index===o;)n.index=null!==p?-1:n.index-h,n=i[s=nextActiveIndexInTemplateParts(i,s)]}l.forEach(t=>t.parentNode.removeChild(t))}var countNodes=t=>{for(var e=11===t.nodeType?0:1,r=document.createTreeWalker(t,walkerNodeFilter,null,!1);r.nextNode();)e++;return e},nextActiveIndexInTemplateParts=function(t){for(var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:-1,r=e+1;r2&&void 0!==arguments[2]?arguments[2]:null,{element:{content:i},parts:a}=t;if(null!=r)for(var s=document.createTreeWalker(i,walkerNodeFilter,null,!1),n=nextActiveIndexInTemplateParts(a),o=0,h=-1;s.nextNode();){h++;var l=s.currentNode;for(l===r&&(o=countNodes(e),r.parentNode.insertBefore(e,r));-1!==n&&a[n].index===h;){if(o>0){for(;-1!==n;)a[n].index+=o,n=nextActiveIndexInTemplateParts(a,n);return}n=nextActiveIndexInTemplateParts(a,n)}}else i.appendChild(e)} +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */var directives=new WeakMap,directive=t=>function(){var e=t(...arguments);return directives.set(e,!0),e},isDirective=t=>"function"==typeof t&&directives.has(t),noChange={},nothing={}; +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */ +class TemplateInstance{constructor(t,e,r){this.__parts=[],this.template=t,this.processor=e,this.options=r}update(t){var e=0;for(var r of this.__parts)void 0!==r&&r.setValue(t[e]),e++;for(var i of this.__parts)void 0!==i&&i.commit()}_clone(){for(var t,e=isCEPolyfill?this.template.element.content.cloneNode(!0):document.importNode(this.template.element.content,!0),r=[],i=this.template.parts,a=document.createTreeWalker(e,133,null,!1),s=0,n=0,o=a.nextNode();st}),commentMarker=" ".concat(marker," ");class TemplateResult{constructor(t,e,r,i){this.strings=t,this.values=e,this.type=r,this.processor=i}getHTML(){for(var t=this.strings.length-1,e="",r=!1,i=0;i-1||r)&&-1===a.indexOf("--\x3e",s+1);var n=lastAttributeNameRegex.exec(a);e+=null===n?a+(r?commentMarker:nodeMarker):a.substr(0,n.index)+n[1]+n[2]+boundAttributeSuffix+n[3]+marker}return e+=this.strings[t]}getTemplateElement(){var t=document.createElement("template"),e=this.getHTML();return void 0!==policy&&(e=policy.createHTML(e)),t.innerHTML=e,t}}class SVGTemplateResult extends TemplateResult{getHTML(){return"".concat(super.getHTML(),"")}getTemplateElement(){var t=super.getTemplateElement(),e=t.content,r=e.firstChild;return e.removeChild(r),reparentNodes(e,r.firstChild),t}} +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */var isPrimitive=t=>null===t||!("object"==typeof t||"function"==typeof t),isIterable=t=>Array.isArray(t)||!(!t||!t[Symbol.iterator]);class AttributeCommitter{constructor(t,e,r){this.dirty=!0,this.element=t,this.name=e,this.strings=r,this.parts=[];for(var i=0;i0&&void 0!==arguments[0]?arguments[0]:this.startNode;removeNodes(this.startNode.parentNode,t.nextSibling,this.endNode)}}class BooleanAttributePart{constructor(t,e,r){if(this.value=void 0,this.__pendingValue=void 0,2!==r.length||""!==r[0]||""!==r[1])throw new Error("Boolean attributes can only contain a single expression");this.element=t,this.name=e,this.strings=r}setValue(t){this.__pendingValue=t}commit(){for(;isDirective(this.__pendingValue);){var t=this.__pendingValue;this.__pendingValue=noChange,t(this)}if(this.__pendingValue!==noChange){var e=!!this.__pendingValue;this.value!==e&&(e?this.element.setAttribute(this.name,""):this.element.removeAttribute(this.name),this.value=e),this.__pendingValue=noChange}}}class PropertyCommitter extends AttributeCommitter{constructor(t,e,r){super(t,e,r),this.single=2===r.length&&""===r[0]&&""===r[1]}_createPart(){return new PropertyPart(this)}_getValue(){return this.single?this.parts[0].value:super._getValue()}commit(){this.dirty&&(this.dirty=!1,this.element[this.name]=this._getValue())}}class PropertyPart extends AttributePart{}var eventOptionsSupported=!1;(()=>{try{var t={get capture(){return eventOptionsSupported=!0,!1}};window.addEventListener("test",t,t),window.removeEventListener("test",t,t)}catch(t){}})();class EventPart{constructor(t,e,r){this.value=void 0,this.__pendingValue=void 0,this.element=t,this.eventName=e,this.eventContext=r,this.__boundHandleEvent=t=>this.handleEvent(t)}setValue(t){this.__pendingValue=t}commit(){for(;isDirective(this.__pendingValue);){var t=this.__pendingValue;this.__pendingValue=noChange,t(this)}if(this.__pendingValue!==noChange){var e=this.__pendingValue,r=this.value,i=null==e||null!=r&&(e.capture!==r.capture||e.once!==r.once||e.passive!==r.passive),a=null!=e&&(null==r||i);i&&this.element.removeEventListener(this.eventName,this.__boundHandleEvent,this.__options),a&&(this.__options=getOptions(e),this.element.addEventListener(this.eventName,this.__boundHandleEvent,this.__options)),this.value=e,this.__pendingValue=noChange}}handleEvent(t){"function"==typeof this.value?this.value.call(this.eventContext||this.element,t):this.value.handleEvent(t)}}var getOptions=t=>t&&(eventOptionsSupported?{capture:t.capture,passive:t.passive,once:t.once}:t.capture) +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */;function templateFactory(t){var e=templateCaches.get(t.type);void 0===e&&(e={stringsArray:new WeakMap,keyString:new Map},templateCaches.set(t.type,e));var r=e.stringsArray.get(t.strings);if(void 0!==r)return r;var i=t.strings.join(marker);return void 0===(r=e.keyString.get(i))&&(r=new Template(t,t.getTemplateElement()),e.keyString.set(i,r)),e.stringsArray.set(t.strings,r),r}var templateCaches=new Map,parts=new WeakMap,render$1=(t,e,r)=>{var i=parts.get(e);void 0===i&&(removeNodes(e,e.firstChild),parts.set(e,i=new NodePart(Object.assign({templateFactory:templateFactory},r))),i.appendInto(e)),i.setValue(t),i.commit()}; +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */ +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */ +class DefaultTemplateProcessor{handleAttributeExpressions(t,e,r,i){var a=e[0];return"."===a?new PropertyCommitter(t,e.slice(1),r).parts:"@"===a?[new EventPart(t,e.slice(1),i.eventContext)]:"?"===a?[new BooleanAttributePart(t,e.slice(1),r)]:new AttributeCommitter(t,e,r).parts}handleTextExpression(t){return new NodePart(t)}}var defaultTemplateProcessor=new DefaultTemplateProcessor; +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */"undefined"!=typeof window&&(window.litHtmlVersions||(window.litHtmlVersions=[])).push("1.3.0");var html=function(t){for(var e=arguments.length,r=new Array(e>1?e-1:0),i=1;i1?e-1:0),i=1;i"".concat(t,"--").concat(e),compatibleShadyCSSVersion=!0;void 0===window.ShadyCSS?compatibleShadyCSSVersion=!1:void 0===window.ShadyCSS.prepareTemplateDom&&(console.warn("Incompatible ShadyCSS version detected. Please update to at least @webcomponents/webcomponentsjs@2.0.2 and @webcomponents/shadycss@1.3.1."),compatibleShadyCSSVersion=!1);var shadyTemplateFactory=t=>e=>{var r=getTemplateCacheKey(e.type,t),i=templateCaches.get(r);void 0===i&&(i={stringsArray:new WeakMap,keyString:new Map},templateCaches.set(r,i));var a=i.stringsArray.get(e.strings);if(void 0!==a)return a;var s=e.strings.join(marker);if(void 0===(a=i.keyString.get(s))){var n=e.getTemplateElement();compatibleShadyCSSVersion&&window.ShadyCSS.prepareTemplateDom(n,t),a=new Template(e,n),i.keyString.set(s,a)}return i.stringsArray.set(e.strings,a),a},TEMPLATE_TYPES=["html","svg"],removeStylesFromLitTemplates=t=>{TEMPLATE_TYPES.forEach(e=>{var r=templateCaches.get(getTemplateCacheKey(e,t));void 0!==r&&r.keyString.forEach(t=>{var{element:{content:e}}=t,r=new Set;Array.from(e.querySelectorAll("style")).forEach(t=>{r.add(t)}),removeNodesFromTemplate(t,r)})})},shadyRenderSet=new Set,prepareTemplateStyles=(t,e,r)=>{shadyRenderSet.add(t);var i=r?r.element:document.createElement("template"),a=e.querySelectorAll("style"),{length:s}=a;if(0!==s){for(var n=document.createElement("style"),o=0;o{if(!r||"object"!=typeof r||!r.scopeName)throw new Error("The `scopeName` option is required.");var i=r.scopeName,a=parts.has(e),s=compatibleShadyCSSVersion&&11===e.nodeType&&!!e.host,n=s&&!shadyRenderSet.has(i),o=n?document.createDocumentFragment():e;if(render$1(t,o,Object.assign({templateFactory:shadyTemplateFactory(i)},r)),n){var h=parts.get(o);parts.delete(o);var l=h.value instanceof TemplateInstance?h.value.template:void 0;prepareTemplateStyles(i,o,l),removeNodes(e,e.firstChild),e.appendChild(o),parts.set(e,h)}!a&&s&&window.ShadyCSS.styleElement(e.host)},_a;window.JSCompiler_renameProperty=(t,e)=>t;var defaultConverter={toAttribute(t,e){switch(e){case Boolean:return t?"":null;case Object:case Array:return null==t?t:JSON.stringify(t)}return t},fromAttribute(t,e){switch(e){case Boolean:return null!==t;case Number:return null===t?null:Number(t);case Object:case Array:return JSON.parse(t)}return t}},notEqual=(t,e)=>e!==t&&(e==e||t==t),defaultPropertyDeclaration={attribute:!0,type:String,converter:defaultConverter,reflect:!1,hasChanged:notEqual},STATE_HAS_UPDATED=1,STATE_UPDATE_REQUESTED=4,STATE_IS_REFLECTING_TO_ATTRIBUTE=8,STATE_IS_REFLECTING_TO_PROPERTY=16,finalized="finalized";class UpdatingElement extends HTMLElement{constructor(){super(),this.initialize()}static get observedAttributes(){this.finalize();var t=[];return this._classProperties.forEach((e,r)=>{var i=this._attributeNameForProperty(r,e);void 0!==i&&(this._attributeToPropertyMap.set(i,r),t.push(i))}),t}static _ensureClassProperties(){if(!this.hasOwnProperty(JSCompiler_renameProperty("_classProperties",this))){this._classProperties=new Map;var t=Object.getPrototypeOf(this)._classProperties;void 0!==t&&t.forEach((t,e)=>this._classProperties.set(e,t))}}static createProperty(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:defaultPropertyDeclaration;if(this._ensureClassProperties(),this._classProperties.set(t,e),!e.noAccessor&&!this.prototype.hasOwnProperty(t)){var r="symbol"==typeof t?Symbol():"__".concat(t),i=this.getPropertyDescriptor(t,r,e);void 0!==i&&Object.defineProperty(this.prototype,t,i)}}static getPropertyDescriptor(t,e,r){return{get(){return this[e]},set(i){var a=this[t];this[e]=i,this.requestUpdateInternal(t,a,r)},configurable:!0,enumerable:!0}}static getPropertyOptions(t){return this._classProperties&&this._classProperties.get(t)||defaultPropertyDeclaration}static finalize(){var t=Object.getPrototypeOf(this);if(t.hasOwnProperty(finalized)||t.finalize(),this[finalized]=!0,this._ensureClassProperties(),this._attributeToPropertyMap=new Map,this.hasOwnProperty(JSCompiler_renameProperty("properties",this))){var e=this.properties,r=[...Object.getOwnPropertyNames(e),..."function"==typeof Object.getOwnPropertySymbols?Object.getOwnPropertySymbols(e):[]];for(var i of r)this.createProperty(i,e[i])}}static _attributeNameForProperty(t,e){var r=e.attribute;return!1===r?void 0:"string"==typeof r?r:"string"==typeof t?t.toLowerCase():void 0}static _valueHasChanged(t,e){return(arguments.length>2&&void 0!==arguments[2]?arguments[2]:notEqual)(t,e)}static _propertyValueFromAttribute(t,e){var r=e.type,i=e.converter||defaultConverter,a="function"==typeof i?i:i.fromAttribute;return a?a(t,r):t}static _propertyValueToAttribute(t,e){if(void 0!==e.reflect){var r=e.type,i=e.converter;return(i&&i.toAttribute||defaultConverter.toAttribute)(t,r)}}initialize(){this._updateState=0,this._updatePromise=new Promise(t=>this._enableUpdatingResolver=t),this._changedProperties=new Map,this._saveInstanceProperties(),this.requestUpdateInternal()}_saveInstanceProperties(){this.constructor._classProperties.forEach((t,e)=>{if(this.hasOwnProperty(e)){var r=this[e];delete this[e],this._instanceProperties||(this._instanceProperties=new Map),this._instanceProperties.set(e,r)}})}_applyInstanceProperties(){this._instanceProperties.forEach((t,e)=>this[e]=t),this._instanceProperties=void 0}connectedCallback(){this.enableUpdating()}enableUpdating(){void 0!==this._enableUpdatingResolver&&(this._enableUpdatingResolver(),this._enableUpdatingResolver=void 0)}disconnectedCallback(){}attributeChangedCallback(t,e,r){e!==r&&this._attributeToProperty(t,r)}_propertyToAttribute(t,e){var r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:defaultPropertyDeclaration,i=this.constructor,a=i._attributeNameForProperty(t,r);if(void 0!==a){var s=i._propertyValueToAttribute(e,r);if(void 0===s)return;this._updateState=this._updateState|STATE_IS_REFLECTING_TO_ATTRIBUTE,null==s?this.removeAttribute(a):this.setAttribute(a,s),this._updateState=this._updateState&~STATE_IS_REFLECTING_TO_ATTRIBUTE}}_attributeToProperty(t,e){if(!(this._updateState&STATE_IS_REFLECTING_TO_ATTRIBUTE)){var r=this.constructor,i=r._attributeToPropertyMap.get(t);if(void 0!==i){var a=r.getPropertyOptions(i);this._updateState=this._updateState|STATE_IS_REFLECTING_TO_PROPERTY,this[i]=r._propertyValueFromAttribute(e,a),this._updateState=this._updateState&~STATE_IS_REFLECTING_TO_PROPERTY}}}requestUpdateInternal(t,e,r){var i=!0;if(void 0!==t){var a=this.constructor;r=r||a.getPropertyOptions(t),a._valueHasChanged(this[t],e,r.hasChanged)?(this._changedProperties.has(t)||this._changedProperties.set(t,e),!0!==r.reflect||this._updateState&STATE_IS_REFLECTING_TO_PROPERTY||(void 0===this._reflectingProperties&&(this._reflectingProperties=new Map),this._reflectingProperties.set(t,r))):i=!1}!this._hasRequestedUpdate&&i&&(this._updatePromise=this._enqueueUpdate())}requestUpdate(t,e){return this.requestUpdateInternal(t,e),this.updateComplete}_enqueueUpdate(){var t=this;return _asyncToGenerator((function*(){t._updateState=t._updateState|STATE_UPDATE_REQUESTED;try{yield t._updatePromise}catch(t){}var e=t.performUpdate();return null!=e&&(yield e),!t._hasRequestedUpdate}))()}get _hasRequestedUpdate(){return this._updateState&STATE_UPDATE_REQUESTED}get hasUpdated(){return this._updateState&STATE_HAS_UPDATED}performUpdate(){if(this._hasRequestedUpdate){this._instanceProperties&&this._applyInstanceProperties();var t=!1,e=this._changedProperties;try{(t=this.shouldUpdate(e))?this.update(e):this._markUpdated()}catch(e){throw t=!1,this._markUpdated(),e}t&&(this._updateState&STATE_HAS_UPDATED||(this._updateState=this._updateState|STATE_HAS_UPDATED,this.firstUpdated(e)),this.updated(e))}}_markUpdated(){this._changedProperties=new Map,this._updateState=this._updateState&~STATE_UPDATE_REQUESTED}get updateComplete(){return this._getUpdateComplete()}_getUpdateComplete(){return this._updatePromise}shouldUpdate(t){return!0}update(t){void 0!==this._reflectingProperties&&this._reflectingProperties.size>0&&(this._reflectingProperties.forEach((t,e)=>this._propertyToAttribute(e,this[e],t)),this._reflectingProperties=void 0),this._markUpdated()}updated(t){}firstUpdated(t){}}_a=finalized,UpdatingElement[_a]=!0; +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */ +var legacyCustomElement=(t,e)=>(window.customElements.define(t,e),e),standardCustomElement=(t,e)=>{var{kind:r,elements:i}=e;return{kind:r,elements:i,finisher(e){window.customElements.define(t,e)}}},customElement=t=>e=>"function"==typeof e?legacyCustomElement(t,e):standardCustomElement(t,e),standardProperty=(t,e)=>"method"===e.kind&&e.descriptor&&!("value"in e.descriptor)?Object.assign(Object.assign({},e),{finisher(r){r.createProperty(e.key,t)}}):{kind:"field",key:Symbol(),placement:"own",descriptor:{},initializer(){"function"==typeof e.initializer&&(this[e.key]=e.initializer.call(this))},finisher(r){r.createProperty(e.key,t)}},legacyProperty=(t,e,r)=>{e.constructor.createProperty(r,t)};function property(t){return(e,r)=>void 0!==r?legacyProperty(t,e,r):standardProperty(t,e)}function internalProperty(t){return property({attribute:!1,hasChanged:null==t?void 0:t.hasChanged})}function query(t,e){return(r,i)=>{var a={get(){return this.renderRoot.querySelector(t)},enumerable:!0,configurable:!0};if(e){var s="symbol"==typeof i?Symbol():"__".concat(i);a.get=function(){return void 0===this[s]&&(this[s]=this.renderRoot.querySelector(t)),this[s]}}return void 0!==i?legacyQuery(a,r,i):standardQuery(a,r)}}function queryAsync(t){return(e,r)=>{var i={get(){var e=this;return _asyncToGenerator((function*(){return yield e.updateComplete,e.renderRoot.querySelector(t)}))()},enumerable:!0,configurable:!0};return void 0!==r?legacyQuery(i,e,r):standardQuery(i,e)}}function queryAll(t){return(e,r)=>{var i={get(){return this.renderRoot.querySelectorAll(t)},enumerable:!0,configurable:!0};return void 0!==r?legacyQuery(i,e,r):standardQuery(i,e)}}var legacyQuery=(t,e,r)=>{Object.defineProperty(e,r,t)},standardQuery=(t,e)=>({kind:"method",placement:"prototype",key:e.key,descriptor:t}),standardEventOptions=(t,e)=>Object.assign(Object.assign({},e),{finisher(r){Object.assign(r.prototype[e.key],t)}}),legacyEventOptions=(t,e,r)=>{Object.assign(e[r],t)};function eventOptions(t){return(e,r)=>void 0!==r?legacyEventOptions(t,e,r):standardEventOptions(t,e)}var ElementProto=Element.prototype,legacyMatches=ElementProto.msMatchesSelector||ElementProto.webkitMatchesSelector;function queryAssignedNodes(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",e=arguments.length>1&&void 0!==arguments[1]&&arguments[1],r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"";return(i,a)=>{var s={get(){var i="slot".concat(t?"[name=".concat(t,"]"):":not([name])"),a=this.renderRoot.querySelector(i),s=a&&a.assignedNodes({flatten:e});return s&&r&&(s=s.filter(t=>t.nodeType===Node.ELEMENT_NODE&&t.matches?t.matches(r):legacyMatches.call(t,r))),s},enumerable:!0,configurable:!0};return void 0!==a?legacyQuery(s,i,a):standardQuery(s,i)}} +/** + @license + Copyright (c) 2019 The Polymer Project Authors. All rights reserved. + This code may only be used under the BSD style license found at + http://polymer.github.io/LICENSE.txt The complete set of authors may be found at + http://polymer.github.io/AUTHORS.txt The complete set of contributors may be + found at http://polymer.github.io/CONTRIBUTORS.txt Code distributed by Google as + part of the polymer project is also subject to an additional IP rights grant + found at http://polymer.github.io/PATENTS.txt + */var supportsAdoptingStyleSheets=window.ShadowRoot&&(void 0===window.ShadyCSS||window.ShadyCSS.nativeShadow)&&"adoptedStyleSheets"in Document.prototype&&"replace"in CSSStyleSheet.prototype,constructionToken=Symbol();class CSSResult{constructor(t,e){if(e!==constructionToken)throw new Error("CSSResult is not constructable. Use `unsafeCSS` or `css` instead.");this.cssText=t}get styleSheet(){return void 0===this._styleSheet&&(supportsAdoptingStyleSheets?(this._styleSheet=new CSSStyleSheet,this._styleSheet.replaceSync(this.cssText)):this._styleSheet=null),this._styleSheet}toString(){return this.cssText}}var unsafeCSS=t=>new CSSResult(String(t),constructionToken),textFromCSSResult=t=>{if(t instanceof CSSResult)return t.cssText;if("number"==typeof t)return t;throw new Error("Value passed to 'css' function must be a 'css' function result: ".concat(t,". Use 'unsafeCSS' to pass non-literal values, but\n take care to ensure page security."))},css=function(t){for(var e=arguments.length,r=new Array(e>1?e-1:0),i=1;ie+textFromCSSResult(r)+t[i+1],t[0]);return new CSSResult(a,constructionToken)}; +/** + * @license + * Copyright (c) 2017 The Polymer Project Authors. All rights reserved. + * This code may only be used under the BSD style license found at + * http://polymer.github.io/LICENSE.txt + * The complete set of authors may be found at + * http://polymer.github.io/AUTHORS.txt + * The complete set of contributors may be found at + * http://polymer.github.io/CONTRIBUTORS.txt + * Code distributed by Google as part of the polymer project is also + * subject to an additional IP rights grant found at + * http://polymer.github.io/PATENTS.txt + */ +(window.litElementVersions||(window.litElementVersions=[])).push("2.4.0");var renderNotImplemented={};class LitElement extends UpdatingElement{static getStyles(){return this.styles}static _getUniqueStyles(){if(!this.hasOwnProperty(JSCompiler_renameProperty("_styles",this))){var t=this.getStyles();if(Array.isArray(t)){var e=(t,r)=>t.reduceRight((t,r)=>Array.isArray(r)?e(r,t):(t.add(r),t),r),r=e(t,new Set),i=[];r.forEach(t=>i.unshift(t)),this._styles=i}else this._styles=void 0===t?[]:[t];this._styles=this._styles.map(t=>{if(t instanceof CSSStyleSheet&&!supportsAdoptingStyleSheets){var e=Array.prototype.slice.call(t.cssRules).reduce((t,e)=>t+e.cssText,"");return unsafeCSS(e)}return t})}}initialize(){super.initialize(),this.constructor._getUniqueStyles(),this.renderRoot=this.createRenderRoot(),window.ShadowRoot&&this.renderRoot instanceof window.ShadowRoot&&this.adoptStyles()}createRenderRoot(){return this.attachShadow({mode:"open"})}adoptStyles(){var t=this.constructor._styles;0!==t.length&&(void 0===window.ShadyCSS||window.ShadyCSS.nativeShadow?supportsAdoptingStyleSheets?this.renderRoot.adoptedStyleSheets=t.map(t=>t instanceof CSSStyleSheet?t:t.styleSheet):this._needsShimAdoptedStyleSheets=!0:window.ShadyCSS.ScopingShim.prepareAdoptedCssText(t.map(t=>t.cssText),this.localName))}connectedCallback(){super.connectedCallback(),this.hasUpdated&&void 0!==window.ShadyCSS&&window.ShadyCSS.styleElement(this)}update(t){var e=this.render();super.update(t),e!==renderNotImplemented&&this.constructor.render(e,this.renderRoot,{scopeName:this.localName,eventContext:this}),this._needsShimAdoptedStyleSheets&&(this._needsShimAdoptedStyleSheets=!1,this.constructor._styles.forEach(t=>{var e=document.createElement("style");e.textContent=t.cssText,this.renderRoot.appendChild(e)}))}render(){return renderNotImplemented}}LitElement.finalized=!0,LitElement.render=render;var commonjsGlobal="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function createCommonjsModule(t,e,r){return t(r={path:e,exports:{},require:function(t,e){return commonjsRequire(t,null==e?r.path:e)}},r.exports),r.exports}function getCjsExportFromNamespace(t){return t&&t.default||t}function commonjsRequire(){throw new Error("Dynamic requires are not currently supported by @rollup/plugin-commonjs")}var lottie=createCommonjsModule((function(module){"undefined"!=typeof navigator&&function(t,e){module.exports?module.exports=e(t):(t.lottie=e(t),t.bodymovin=t.lottie)}(window||{},(function(window){var svgNS="http://www.w3.org/2000/svg",locationHref="",initialDefaultFrame=-999999,subframeEnabled=!0,expressionsPlugin,isSafari=/^((?!chrome|android).)*safari/i.test(navigator.userAgent),cachedColors={},bmRnd,bmPow=Math.pow,bmSqrt=Math.sqrt,bmFloor=Math.floor,bmMax=Math.max,bmMin=Math.min,BMMath={};function ProjectInterface(){return{}}!function(){var t,e=["abs","acos","acosh","asin","asinh","atan","atanh","atan2","ceil","cbrt","expm1","clz32","cos","cosh","exp","floor","fround","hypot","imul","log","log1p","log2","log10","max","min","pow","random","round","sign","sin","sinh","sqrt","tan","tanh","trunc","E","LN10","LN2","LOG10E","LOG2E","PI","SQRT1_2","SQRT2"],r=e.length;for(t=0;t1?r[1]=1:r[1]<=0&&(r[1]=0),HSVtoRGB(r[0],r[1],r[2])}function addBrightnessToRGB(t,e){var r=RGBtoHSV(255*t[0],255*t[1],255*t[2]);return r[2]+=e,r[2]>1?r[2]=1:r[2]<0&&(r[2]=0),HSVtoRGB(r[0],r[1],r[2])}function addHueToRGB(t,e){var r=RGBtoHSV(255*t[0],255*t[1],255*t[2]);return r[0]+=e/360,r[0]>1?r[0]-=1:r[0]<0&&(r[0]+=1),HSVtoRGB(r[0],r[1],r[2])}var rgbToHex=function(){var t,e,r=[];for(t=0;t<256;t+=1)e=t.toString(16),r[t]=1===e.length?"0"+e:e;return function(t,e,i){return t<0&&(t=0),e<0&&(e=0),i<0&&(i=0),"#"+r[t]+r[e]+r[i]}}();function BaseEvent(){}BaseEvent.prototype={triggerEvent:function(t,e){if(this._cbs[t])for(var r=this._cbs[t].length,i=0;i0||t>-1e-6&&t<0?i(1e4*t)/1e4:t}function F(){var t=this.props;return"matrix("+M(t[0])+","+M(t[1])+","+M(t[4])+","+M(t[5])+","+M(t[12])+","+M(t[13])+")"}return function(){this.reset=a,this.rotate=s,this.rotateX=n,this.rotateY=o,this.rotateZ=h,this.skew=p,this.skewFromAxis=c,this.shear=l,this.scale=f,this.setTransform=d,this.translate=m,this.transform=u,this.applyToPoint=_,this.applyToX=P,this.applyToY=S,this.applyToZ=E,this.applyToPointArray=T,this.applyToTriplePoints=C,this.applyToPointStringified=k,this.toCSS=D,this.to2dCSS=F,this.clone=v,this.cloneFromProps=b,this.equals=g,this.inversePoints=w,this.inversePoint=A,this.getInverseMatrix=x,this._t=this.transform,this.isIdentity=y,this._identity=!0,this._identityCalculated=!1,this.props=createTypedArray("float32",16),this.reset()}}(); +/*! + Transformation Matrix v2.0 + (c) Epistemex 2014-2015 + www.epistemex.com + By Ken Fyrstenberg + Contributions by leeoniya. + License: MIT, header required. + */!function(t,e){var r=this,i=e.pow(256,6),a=e.pow(2,52),s=2*a;function n(t){var e,r=t.length,i=this,a=0,s=i.i=i.j=0,n=i.S=[];for(r||(t=[r++]);a<256;)n[a]=a++;for(a=0;a<256;a++)n[a]=n[s=255&s+t[a%r]+(e=n[a])],n[s]=e;i.g=function(t){for(var e,r=0,a=i.i,s=i.j,n=i.S;t--;)e=n[a=255&a+1],r=256*r+n[255&(n[a]=n[s=255&s+e])+(n[s]=e)];return i.i=a,i.j=s,r}}function o(t,e){return e.i=t.i,e.j=t.j,e.S=t.S.slice(),e}function h(t,e){for(var r,i=t+"",a=0;a=s;)t/=2,e/=2,r>>>=1;return(t+r)/e};return y.int32=function(){return 0|u.g(4)},y.quick=function(){return u.g(4)/4294967296},y.double=y,h(l(u.S),t),(c.pass||f||function(t,r,i,a){return a&&(a.S&&o(a,u),t.state=function(){return o(u,{})}),i?(e.random=t,r):t})(y,m,"global"in c?c.global:this==e,c.state)},h(e.random(),t)}([],BMMath);var BezierFactory=function(){var t={getBezierEasing:function(t,r,i,a,s){var n=s||("bez_"+t+"_"+r+"_"+i+"_"+a).replace(/\./g,"p");if(e[n])return e[n];var o=new h([t,r,i,a]);return e[n]=o,o}},e={};var r="function"==typeof Float32Array;function i(t,e){return 1-3*e+3*t}function a(t,e){return 3*e-6*t}function s(t){return 3*t}function n(t,e,r){return((i(e,r)*t+a(e,r))*t+s(e))*t}function o(t,e,r){return 3*i(e,r)*t*t+2*a(e,r)*t+s(e)}function h(t){this._p=t,this._mSampleValues=r?new Float32Array(11):new Array(11),this._precomputed=!1,this.get=this.get.bind(this)}return h.prototype={get:function(t){var e=this._p[0],r=this._p[1],i=this._p[2],a=this._p[3];return this._precomputed||this._precompute(),e===r&&i===a?t:0===t?0:1===t?1:n(this._getTForX(t),r,a)},_precompute:function(){var t=this._p[0],e=this._p[1],r=this._p[2],i=this._p[3];this._precomputed=!0,t===e&&r===i||this._calcSampleValues()},_calcSampleValues:function(){for(var t=this._p[0],e=this._p[2],r=0;r<11;++r)this._mSampleValues[r]=n(.1*r,t,e)},_getTForX:function(t){for(var e=this._p[0],r=this._p[2],i=this._mSampleValues,a=0,s=1;10!==s&&i[s]<=t;++s)a+=.1;var h=a+.1*((t-i[--s])/(i[s+1]-i[s])),l=o(h,e,r);return l>=.001?function(t,e,r,i){for(var a=0;a<4;++a){var s=o(e,r,i);if(0===s)return e;e-=(n(e,r,i)-t)/s}return e}(t,h,e,r):0===l?h:function(t,e,r,i,a){var s,o,h=0;do{(s=n(o=e+(r-e)/2,i,a)-t)>0?r=o:e=o}while(Math.abs(s)>1e-7&&++h<10);return o}(t,a,a+.1,e,r)}},t}();function extendPrototype(t,e){var r,i,a=t.length;for(r=0;r-.001&&n<.001}var r=function(t,e,r,i){var a,s,n,o,h,l,p=defaultCurveSegments,c=0,f=[],d=[],m=bezierLengthPool.newElement();for(n=r.length,a=0;an?-1:1,l=!0;l;)if(i[s]<=n&&i[s+1]>n?(o=(n-i[s])/(i[s+1]-i[s]),l=!1):s+=h,s<0||s>=a-1){if(s===a-1)return r[s];l=!1}return r[s]+(r[s+1]-r[s])*o}var h=createTypedArray("float32",8);return{getSegmentsLength:function(t){var e,i=segmentsLengthPool.newElement(),a=t.c,s=t.v,n=t.o,o=t.i,h=t._length,l=i.lengths,p=0;for(e=0;e1&&(s=1);var p,c=o(s,l),f=o(n=n>1?1:n,l),d=e.length,m=1-c,u=1-f,y=m*m*m,g=c*m*m*3,v=c*c*m*3,b=c*c*c,_=m*m*u,P=c*m*u+m*c*u+m*m*f,S=c*c*u+m*c*f+c*m*f,E=c*c*f,x=m*u*u,A=c*u*u+m*f*u+m*u*f,w=c*f*u+m*f*f+c*u*f,C=c*f*f,T=u*u*u,k=f*u*u+u*f*u+u*u*f,D=f*f*u+u*f*f+f*u*f,M=f*f*f;for(p=0;pd?f>m?f-d-m:m-d-f:m>d?m-d-f:d-f-m)>-1e-4&&c<1e-4}}}!function(){for(var t=0,e=["ms","moz","webkit","o"],r=0;r=0;e-=1)if("sh"===t[e].ty)if(t[e].ks.k.i)i(t[e].ks.k);else for(s=t[e].ks.k.length,a=0;ar[0]||!(r[0]>t[0])&&(t[1]>r[1]||!(r[1]>t[1])&&(t[2]>r[2]||!(r[2]>t[2])&&null))}var s,n=function(){var t=[4,4,14];function e(t){var e,r,i,a=t.length;for(e=0;e=0;r-=1)if("sh"===t[r].ty)if(t[r].ks.k.i)t[r].ks.k.c=t[r].closed;else for(a=t[r].ks.k.length,i=0;i0&&(p=!1),p){var c=createTag("style");c.setAttribute("f-forigin",s[a].fOrigin),c.setAttribute("f-origin",s[a].origin),c.setAttribute("f-family",s[a].fFamily),c.type="text/css",c.innerText="@font-face {font-family: "+s[a].fFamily+"; font-style: normal; src: url('"+s[a].fPath+"');}",e.appendChild(c)}}else if("g"===s[a].fOrigin||1===s[a].origin){for(h=document.querySelectorAll('link[f-forigin="g"], link[f-origin="1"]'),l=0;l=n.t-a){s.h&&(s=n),d=0;break}if(n.t-a>t){d=m;break}m=v||t=v?_.points.length-1:0;for(h=_.points[P].point.length,o=0;o=x&&E=v)r[0]=g[0],r[1]=g[1],r[2]=g[2];else if(t<=b)r[0]=s.s[0],r[1]=s.s[1],r[2]=s.s[2];else{!function(t,e){var r=e[0],i=e[1],a=e[2],s=e[3],n=Math.atan2(2*i*s-2*r*a,1-2*i*i-2*a*a),o=Math.asin(2*r*i+2*a*s),h=Math.atan2(2*r*s-2*i*a,1-2*r*r-2*a*a);t[0]=n/degToRads,t[1]=o/degToRads,t[2]=h/degToRads}(r,function(t,e,r){var i,a,s,n,o,h=[],l=t[0],p=t[1],c=t[2],f=t[3],d=e[0],m=e[1],u=e[2],y=e[3];(a=l*d+p*m+c*u+f*y)<0&&(a=-a,d=-d,m=-m,u=-u,y=-y);1-a>1e-6?(i=Math.acos(a),s=Math.sin(i),n=Math.sin((1-r)*i)/s,o=Math.sin(r*i)/s):(n=1-r,o=r);return h[0]=n*l+o*d,h[1]=n*p+o*m,h[2]=n*c+o*u,h[3]=n*f+o*y,h}(i(s.s),i(g),(t-b)/(v-b)))}else for(m=0;m=v?l=1:t=i&&e>=i||this._caching.lastFrame=e&&(this._caching._lastKeyframeIndex=-1,this._caching.lastIndex=0);var a=this.interpolateValue(e,this._caching);this.pv=a}return this._caching.lastFrame=e,this.pv}function s(t){var r;if("unidimensional"===this.propType)r=t*this.mult,e(this.v-r)>1e-5&&(this.v=r,this._mdf=!0);else for(var i=0,a=this.v.length;i1e-5&&(this.v[i]=r,this._mdf=!0),i+=1}function n(){if(this.elem.globalData.frameId!==this.frameId&&this.effectsSequence.length)if(this.lock)this.setVValue(this.pv);else{var t;this.lock=!0,this._mdf=this._isFirstFrame;var e=this.effectsSequence.length,r=this.kf?this.pv:this.data.k;for(t=0;t=this.p.keyframes[this.p.keyframes.length-1].t?(i=this.p.getValueAtTime(this.p.keyframes[this.p.keyframes.length-1].t/r,0),a=this.p.getValueAtTime((this.p.keyframes[this.p.keyframes.length-1].t-.05)/r,0)):(i=this.p.pv,a=this.p.getValueAtTime((this.p._caching.lastFrame+this.p.offsetTime-.01)/r,this.p.offsetTime));else if(this.px&&this.px.keyframes&&this.py.keyframes&&this.px.getValueAtTime&&this.py.getValueAtTime){i=[],a=[];var s=this.px,n=this.py;s._caching.lastFrame+s.offsetTime<=s.keyframes[0].t?(i[0]=s.getValueAtTime((s.keyframes[0].t+.01)/r,0),i[1]=n.getValueAtTime((n.keyframes[0].t+.01)/r,0),a[0]=s.getValueAtTime(s.keyframes[0].t/r,0),a[1]=n.getValueAtTime(n.keyframes[0].t/r,0)):s._caching.lastFrame+s.offsetTime>=s.keyframes[s.keyframes.length-1].t?(i[0]=s.getValueAtTime(s.keyframes[s.keyframes.length-1].t/r,0),i[1]=n.getValueAtTime(n.keyframes[n.keyframes.length-1].t/r,0),a[0]=s.getValueAtTime((s.keyframes[s.keyframes.length-1].t-.01)/r,0),a[1]=n.getValueAtTime((n.keyframes[n.keyframes.length-1].t-.01)/r,0)):(i=[s.pv,n.pv],a[0]=s.getValueAtTime((s._caching.lastFrame+s.offsetTime-.01)/r,s.offsetTime),a[1]=n.getValueAtTime((n._caching.lastFrame+n.offsetTime-.01)/r,n.offsetTime))}else i=a=t;this.v.rotate(-Math.atan2(i[1]-a[1],i[0]-a[0]))}this.data.p&&this.data.p.s?this.data.p.z?this.v.translate(this.px.v,this.py.v,-this.pz.v):this.v.translate(this.px.v,this.py.v,0):this.v.translate(this.p.v[0],this.p.v[1],-this.p.v[2])}this.frameId=this.elem.globalData.frameId}},precalculateMatrix:function(){if(!this.a.k&&(this.pre.translate(-this.a.v[0],-this.a.v[1],this.a.v[2]),this.appliedTransformations=1,!this.s.effectsSequence.length)){if(this.pre.scale(this.s.v[0],this.s.v[1],this.s.v[2]),this.appliedTransformations=2,this.sk){if(this.sk.effectsSequence.length||this.sa.effectsSequence.length)return;this.pre.skewFromAxis(-this.sk.v,this.sa.v),this.appliedTransformations=3}this.r?this.r.effectsSequence.length||(this.pre.rotate(-this.r.v),this.appliedTransformations=4):this.rz.effectsSequence.length||this.ry.effectsSequence.length||this.rx.effectsSequence.length||this.or.effectsSequence.length||(this.pre.rotateZ(-this.rz.v).rotateY(this.ry.v).rotateX(this.rx.v).rotateZ(-this.or.v[2]).rotateY(this.or.v[1]).rotateX(this.or.v[0]),this.appliedTransformations=4)}},autoOrient:function(){}},extendPrototype([DynamicPropertyContainer],e),e.prototype.addDynamicProperty=function(t){this._addDynamicProperty(t),this.elem.addDynamicProperty(t),this._isDirty=!0},e.prototype._addDynamicProperty=DynamicPropertyContainer.prototype.addDynamicProperty,{getTransformProperty:function(t,r,i){return new e(t,r,i)}}}();function ShapePath(){this.c=!1,this._length=0,this._maxLength=8,this.v=createSizedArray(this._maxLength),this.o=createSizedArray(this._maxLength),this.i=createSizedArray(this._maxLength)}ShapePath.prototype.setPathData=function(t,e){this.c=t,this.setLength(e);for(var r=0;r=this._maxLength&&this.doubleArrayLength(),r){case"v":s=this.v;break;case"i":s=this.i;break;case"o":s=this.o;break;default:s=[]}(!s[i]||s[i]&&!a)&&(s[i]=pointPool.newElement()),s[i][0]=t,s[i][1]=e},ShapePath.prototype.setTripleAt=function(t,e,r,i,a,s,n,o){this.setXYAt(t,e,"v",n,o),this.setXYAt(r,i,"o",n,o),this.setXYAt(a,s,"i",n,o)},ShapePath.prototype.reverse=function(){var t=new ShapePath;t.setPathData(this.c,this._length);var e=this.v,r=this.o,i=this.i,a=0;this.c&&(t.setTripleAt(e[0][0],e[0][1],i[0][0],i[0][1],r[0][0],r[0][1],0,!1),a=1);var s,n=this._length-1,o=this._length;for(s=a;s=d[d.length-1].t-this.offsetTime)i=d[d.length-1].s?d[d.length-1].s[0]:d[d.length-2].e[0],s=!0;else{for(var m,u,y=f,g=d.length-1,v=!0;v&&(m=d[y],!((u=d[y+1]).t-this.offsetTime>t));)y=u.t-this.offsetTime)p=1;else if(tr&&t>r)||(this._caching.lastIndex=i=1?s.push({s:t-1,e:e-1}):(s.push({s:t,e:1}),s.push({s:0,e:e-1}));var n,o,h=[],l=s.length;for(n=0;ni+r))p=o.s*a<=i?0:(o.s*a-i)/r,c=o.e*a>=i+r?1:(o.e*a-i)/r,h.push([p,c])}return h.length||h.push([0,0]),h},TrimModifier.prototype.releasePathsData=function(t){var e,r=t.length;for(e=0;e1?1+s:this.s.v<0?0+s:this.s.v+s)>(r=this.e.v>1?1+s:this.e.v<0?0+s:this.e.v+s)){var n=e;e=r,r=n}e=1e-4*Math.round(1e4*e),r=1e-4*Math.round(1e4*r),this.sValue=e,this.eValue=r}else e=this.sValue,r=this.eValue;var o,h,l,p,c,f=this.shapes.length,d=0;if(r===e)for(a=0;a=0;a-=1)if((m=this.shapes[a]).shape._mdf){for((u=m.localShapeCollection).releaseShapes(),2===this.m&&f>1?(g=this.calculateShapeEdges(e,r,m.totalShapeLength,_,d),_+=m.totalShapeLength):g=[[v,b]],h=g.length,o=0;o=1?y.push({s:m.totalShapeLength*(v-1),e:m.totalShapeLength*(b-1)}):(y.push({s:m.totalShapeLength*v,e:m.totalShapeLength}),y.push({s:0,e:m.totalShapeLength*(b-1)}));var P=this.addShapes(m,y[0]);if(y[0].s!==y[0].e){if(y.length>1)if(m.shape.paths.shapes[m.shape.paths._length-1].c){var S=P.pop();this.addPaths(P,u),P=this.addShapes(m,y[1],S)}else this.addPaths(P,u),P=this.addShapes(m,y[1]);this.addPaths(P,u)}}m.shape.paths=u}}},TrimModifier.prototype.addPaths=function(t,e){var r,i=t.length;for(r=0;re.e){r.c=!1;break}e.s<=m&&e.e>=m+n.addedLength?(this.addSegment(f[i].v[a-1],f[i].o[a-1],f[i].i[a],f[i].v[a],r,o,y),y=!1):(l=bez.getNewSegment(f[i].v[a-1],f[i].v[a],f[i].o[a-1],f[i].i[a],(e.s-m)/n.addedLength,(e.e-m)/n.addedLength,h[a-1]),this.addSegmentFromArray(l,r,o,y),y=!1,r.c=!1),m+=n.addedLength,o+=1}if(f[i].c&&h.length){if(n=h[a-1],m<=e.e){var g=h[a-1].addedLength;e.s<=m&&e.e>=m+g?(this.addSegment(f[i].v[a-1],f[i].o[a-1],f[i].i[0],f[i].v[0],r,o,y),y=!1):(l=bez.getNewSegment(f[i].v[a-1],f[i].v[0],f[i].o[a-1],f[i].i[0],(e.s-m)/g,(e.e-m)/g,h[a-1]),this.addSegmentFromArray(l,r,o,y),y=!1,r.c=!1)}else r.c=!1;m+=n.addedLength,o+=1}if(r._length&&(r.setXYAt(r.v[p][0],r.v[p][1],"i",p),r.setXYAt(r.v[r._length-1][0],r.v[r._length-1][1],"o",r._length-1)),m>e.e)break;i0;)r-=1,this._elements.unshift(e[r]);this.dynamicProperties.length?this.k=!0:this.getValue(!0)},RepeaterModifier.prototype.resetElements=function(t){var e,r=t.length;for(e=0;e0?Math.floor(f):Math.ceil(f),u=this.pMatrix.props,y=this.rMatrix.props,g=this.sMatrix.props;this.pMatrix.reset(),this.rMatrix.reset(),this.sMatrix.reset(),this.tMatrix.reset(),this.matrix.reset();var v,b,_=0;if(f>0){for(;_m;)this.applyTransforms(this.pMatrix,this.rMatrix,this.sMatrix,this.tr,1,!0),_-=1;d&&(this.applyTransforms(this.pMatrix,this.rMatrix,this.sMatrix,this.tr,-d,!0),_-=d)}for(i=1===this.data.m?0:this._currentCopies-1,a=1===this.data.m?1:-1,s=this._currentCopies;s;){if(b=(r=(e=this.elemsData[i].it)[e.length-1].transform.mProps.v.props).length,e[e.length-1].transform.mProps._mdf=!0,e[e.length-1].transform.op._mdf=!0,e[e.length-1].transform.op.v=1===this._currentCopies?this.so.v:this.so.v+(this.eo.v-this.so.v)*(i/(this._currentCopies-1)),0!==_){for((0!==i&&1===a||i!==this._currentCopies-1&&-1===a)&&this.applyTransforms(this.pMatrix,this.rMatrix,this.sMatrix,this.tr,1,!1),this.matrix.transform(y[0],y[1],y[2],y[3],y[4],y[5],y[6],y[7],y[8],y[9],y[10],y[11],y[12],y[13],y[14],y[15]),this.matrix.transform(g[0],g[1],g[2],g[3],g[4],g[5],g[6],g[7],g[8],g[9],g[10],g[11],g[12],g[13],g[14],g[15]),this.matrix.transform(u[0],u[1],u[2],u[3],u[4],u[5],u[6],u[7],u[8],u[9],u[10],u[11],u[12],u[13],u[14],u[15]),v=0;v.01)return!1;r+=1}return!0},GradientProperty.prototype.checkCollapsable=function(){if(this.o.length/2!=this.c.length/4)return!1;if(this.data.k.k[0].s)for(var t=0,e=this.data.k.k.length;t500)&&(this._imageLoaded(),clearInterval(r)),e+=1}.bind(this),50)}function s(t){var e={assetData:t},r=i(t,this.assetsPath,this.path);return assetLoader.load(r,function(t){e.img=t,this._footageLoaded()}.bind(this),function(){e.img={},this._footageLoaded()}.bind(this)),e}function n(){this._imageLoaded=e.bind(this),this._footageLoaded=r.bind(this),this.testImageLoaded=a.bind(this),this.createFootageData=s.bind(this),this.assetsPath="",this.path="",this.totalImages=0,this.totalFootages=0,this.loadedAssets=0,this.loadedFootagesCount=0,this.imagesLoadedCb=null,this.images=[]}return n.prototype={loadAssets:function(t,e){var r;this.imagesLoadedCb=e;var i=t.length;for(r=0;r=o+ot||!m?(v=(o+ot-l)/h.partialLength,B=d.point[0]+(h.point[0]-d.point[0])*v,N=d.point[1]+(h.point[1]-d.point[1])*v,x.translate(-P[0]*C[a].an*.005,-P[1]*V*.01),p=!1):m&&(l+=h.partialLength,(c+=1)>=m.length&&(c=0,u[f+=1]?m=u[f].points:_.v.c?(c=0,m=u[f=0].points):(l-=h.partialLength,m=null)),m&&(d=h,y=(h=m[c]).partialLength));L=C[a].an/2-C[a].add,x.translate(-L,0,0)}else L=C[a].an/2-C[a].add,x.translate(-L,0,0),x.translate(-P[0]*C[a].an*.005,-P[1]*V*.01,0);for(M=0;M1,this.kf&&this.addEffect(this.getKeyframeValue.bind(this)),this.kf},TextProperty.prototype.addEffect=function(t){this.effectsSequence.push(t),this.elem.addDynamicProperty(this)},TextProperty.prototype.getValue=function(t){if(this.elem.globalData.frameId!==this.frameId&&this.effectsSequence.length||t){this.currentData.t=this.data.d.k[this.keysIndex].s.t;var e=this.currentData,r=this.keysIndex;if(this.lock)this.setCurrentData(this.currentData);else{var i;this.lock=!0,this._mdf=!1;var a=this.effectsSequence.length,s=t||this.data.d.k[this.keysIndex].s;for(i=0;ie);)r+=1;return this.keysIndex!==r&&(this.keysIndex=r),this.data.d.k[this.keysIndex].s},TextProperty.prototype.buildFinalText=function(t){for(var e,r=FontManager.getCombinedCharacterCodes(),i=[],a=0,s=t.length;a=55296&&e<=56319&&(e=t.charCodeAt(a+1))>=56320&&e<=57343?(i.push(t.substr(a,2)),a+=1):i.push(t.charAt(a)),a+=1;return i},TextProperty.prototype.completeTextData=function(t){t.__complete=!0;var e,r,i,a,s,n,o,h=this.elem.globalData.fontManager,l=this.data,p=[],c=0,f=l.m.g,d=0,m=0,u=0,y=[],g=0,v=0,b=h.getFontByName(t.f),_=0,P=getFontProperties(b);t.fWeight=P.weight,t.fStyle=P.style,t.finalSize=t.s,t.finalText=this.buildFinalText(t.t),r=t.finalText.length,t.finalLineHeight=t.lh;var S,E=t.tr/1e3*t.finalSize;if(t.sz)for(var x,A,w=!0,C=t.sz[0],T=t.sz[1];w;){x=0,g=0,r=(A=this.buildFinalText(t.t)).length,E=t.tr/1e3*t.finalSize;var k=-1;for(e=0;eC&&" "!==A[e]?(-1===k?r+=1:e=k,x+=t.finalLineHeight||1.2*t.finalSize,A.splice(e,k===e?1:0,"\r"),k=-1,g=0):(g+=_,g+=E);x+=b.ascent*t.finalSize/100,this.canResize&&t.finalSize>this.minimumFontSize&&Tv?g:v,g=-2*E,a="",i=!0,u+=1):a=D,h.chars?(o=h.getCharData(D,b.fStyle,h.getFontByName(t.f).fFamily),_=i?0:o.w*t.finalSize/100):_=h.measureText(a,t.f,t.finalSize)," "===D?M+=_+E:(g+=_+E+M,M=0),p.push({l:_,an:_,add:d,n:i,anIndexes:[],val:a,line:u,animatorJustifyOffset:0}),2==f){if(d+=_,""===a||" "===a||e===r-1){for(""!==a&&" "!==a||(d-=_);m<=e;)p[m].an=d,p[m].ind=c,p[m].extra=_,m+=1;c+=1,d=0}}else if(3==f){if(d+=_,""===a||e===r-1){for(""===a&&(d-=_);m<=e;)p[m].an=d,p[m].ind=c,p[m].extra=_,m+=1;d=0,c+=1}}else p[c].ind=c,p[c].extra=0,c+=1;if(t.l=p,v=g>v?g:v,y.push(g),t.sz)t.boxWidth=t.sz[0],t.justifyOffset=0;else switch(t.boxWidth=v,t.j){case 1:t.justifyOffset=-t.boxWidth;break;case 2:t.justifyOffset=-t.boxWidth/2;break;default:t.justifyOffset=0}t.lineWidths=y;var F,I,R,V,O=l.a;n=O.length;var L=[];for(s=0;s0?a=this.ne.v/100:s=-this.ne.v/100,this.xe.v>0?n=1-this.xe.v/100:o=1+this.xe.v/100;var h=BezierFactory.getBezierEasing(a,s,n,o).get,l=0,p=this.finalS,c=this.finalE,f=this.data.sh;if(2===f)l=h(l=c===p?i>=c?1:0:t(0,e(.5/(c-p)+(i-p)/(c-p),1)));else if(3===f)l=h(l=c===p?i>=c?0:1:1-t(0,e(.5/(c-p)+(i-p)/(c-p),1)));else if(4===f)c===p?l=0:(l=t(0,e(.5/(c-p)+(i-p)/(c-p),1)))<.5?l*=2:l=1-2*(l-.5),l=h(l);else if(5===f){if(c===p)l=0;else{var d=c-p,m=-d/2+(i=e(t(0,i+.5-p),c-p)),u=d/2;l=Math.sqrt(1-m*m/(u*u))}l=h(l)}else 6===f?(c===p?l=0:(i=e(t(0,i+.5-p),c-p),l=(1+Math.cos(Math.PI+2*Math.PI*i/(c-p)))/2),l=h(l)):(i>=r(p)&&(l=t(0,e(i-p<0?e(c,1)-(p-i):c-i,1))),l=h(l));return l*this.a.v},getValue:function(t){this.iterateDynamicProperties(),this._mdf=t||this._mdf,this._currentTextLength=this.elem.textProperty.currentData.l.length||0,t&&2===this.data.r&&(this.e.v=this._currentTextLength);var e=2===this.data.r?1:100/this.data.totalChars,r=this.o.v/e,i=this.s.v/e+r,a=this.e.v/e+r;if(i>a){var s=i;i=a,a=s}this.finalS=i,this.finalE=a}},extendPrototype([DynamicPropertyContainer],i),{getTextSelectorProp:function(t,e,r){return new i(t,e,r)}}}(),poolFactory=function(t,e,r){var i=0,a=t,s=createSizedArray(a);return{newElement:function(){return i?s[i-=1]:e()},release:function(t){i===a&&(s=pooling.double(s),a*=2),r&&r(t),s[i]=t,i+=1}}},pooling={double:function(t){return t.concat(createSizedArray(t.length))}},pointPool=poolFactory(8,(function(){return createTypedArray("float32",2)})),shapePool=(factory=poolFactory(4,(function(){return new ShapePath}),(function(t){var e,r=t._length;for(e=0;e0&&(this.maskElement.setAttribute("id",y),this.element.maskedElement.setAttribute(v,"url("+locationHref+"#"+y+")"),s.appendChild(this.maskElement)),this.viewData.length&&this.element.addRenderableComponent(this)}function HierarchyElement(){}function FrameElement(){}function TransformElement(){}function RenderableElement(){}function RenderableDOMElement(){}function ProcessedElement(t,e){this.elem=t,this.pos=e}function SVGStyleData(t,e){this.data=t,this.type=t.ty,this.d="",this.lvl=e,this._mdf=!1,this.closed=!0===t.hd,this.pElem=createNS("path"),this.msElem=null}function SVGShapeData(t,e,r){this.caches=[],this.styles=[],this.transformers=t,this.lStr="",this.sh=r,this.lvl=e,this._isAnimated=!!r.k;for(var i=0,a=t.length;i=0;e-=1)this.elements[e]||(r=this.layers[e]).ip-r.st<=t-this.layers[e].st&&r.op-r.st>t-this.layers[e].st&&this.buildItem(e),this.completeLayers=!!this.elements[e]&&this.completeLayers;this.checkPendingElements()},BaseRenderer.prototype.createItem=function(t){switch(t.ty){case 2:return this.createImage(t);case 0:return this.createComp(t);case 1:return this.createSolid(t);case 3:return this.createNull(t);case 4:return this.createShape(t);case 5:return this.createText(t);case 6:return this.createAudio(t);case 13:return this.createCamera(t);case 15:return this.createFootage(t);default:return this.createNull(t)}},BaseRenderer.prototype.createCamera=function(){throw new Error("You're using a 3d camera. Try the html renderer.")},BaseRenderer.prototype.createAudio=function(t){return new AudioElement(t,this.globalData,this)},BaseRenderer.prototype.createFootage=function(t){return new FootageElement(t,this.globalData,this)},BaseRenderer.prototype.buildAllItems=function(){var t,e=this.layers.length;for(t=0;t=0;e-=1)(this.completeLayers||this.elements[e])&&this.elements[e].prepareFrame(t-this.layers[e].st);if(this.globalData._mdf)for(e=0;er&&"meet"===s||ir&&"slice"===s)?(t-this.transformCanvas.w*(e/this.transformCanvas.h))/2*this.renderConfig.dpr:"xMax"===o&&(ir&&"slice"===s)?(t-this.transformCanvas.w*(e/this.transformCanvas.h))*this.renderConfig.dpr:0,this.transformCanvas.ty="YMid"===h&&(i>r&&"meet"===s||ir&&"meet"===s||i=0;t-=1)this.elements[t]&&this.elements[t].destroy();this.elements.length=0,this.globalData.canvasContext=null,this.animationItem.container=null,this.destroyed=!0},CanvasRenderer.prototype.renderFrame=function(t,e){if((this.renderedFrame!==t||!0!==this.renderConfig.clearCanvas||e)&&!this.destroyed&&-1!==t){var r;this.renderedFrame=t,this.globalData.frameNum=t-this.animationItem._isFirstFrame,this.globalData.frameId+=1,this.globalData._mdf=!this.renderConfig.clearCanvas||e,this.globalData.projectInterface.currentFrame=t;var i=this.layers.length;for(this.completeLayers||this.checkLayers(t),r=0;r=0;r-=1)(this.completeLayers||this.elements[r])&&this.elements[r].renderFrame();!0!==this.renderConfig.clearCanvas&&this.restore()}}},CanvasRenderer.prototype.buildItem=function(t){var e=this.elements;if(!e[t]&&99!==this.layers[t].ty){var r=this.createItem(this.layers[t],this,this.globalData);e[t]=r,r.initExpressions()}},CanvasRenderer.prototype.checkPendingElements=function(){for(;this.pendingElements.length;){this.pendingElements.pop().checkParenting()}},CanvasRenderer.prototype.hide=function(){this.animationItem.container.style.display="none"},CanvasRenderer.prototype.show=function(){this.animationItem.container.style.display="block"},extendPrototype([BaseRenderer],HybridRenderer),HybridRenderer.prototype.buildItem=SVGRenderer.prototype.buildItem,HybridRenderer.prototype.checkPendingElements=function(){for(;this.pendingElements.length;){this.pendingElements.pop().checkParenting()}},HybridRenderer.prototype.appendElementInPos=function(t,e){var r=t.getBaseElement();if(r){var i=this.layers[e];if(i.ddd&&this.supports3d)this.addTo3dContainer(r,e);else if(this.threeDElements)this.addTo3dContainer(r,e);else{for(var a,s,n=0;n=t)return this.threeDElements[e].perspectiveElem;e+=1}return null},HybridRenderer.prototype.createThreeDContainer=function(t,e){var r,i,a=createTag("div");styleDiv(a);var s=createTag("div");if(styleDiv(s),"3d"===e){(r=a.style).width=this.globalData.compSize.w+"px",r.height=this.globalData.compSize.h+"px";r.webkitTransformOrigin="50% 50%",r.mozTransformOrigin="50% 50%",r.transformOrigin="50% 50%";var n="matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1)";(i=s.style).transform=n,i.webkitTransform=n}a.appendChild(s);var o={container:s,perspectiveElem:a,startPos:t,endPos:t,type:e};return this.threeDElements.push(o),o},HybridRenderer.prototype.build3dContainers=function(){var t,e,r=this.layers.length,i="";for(t=0;t=0;t-=1)this.resizerElem.appendChild(this.threeDElements[t].perspectiveElem)},HybridRenderer.prototype.addTo3dContainer=function(t,e){for(var r=0,i=this.threeDElements.length;rn?(t=a/this.globalData.compSize.w,e=a/this.globalData.compSize.w,r=0,i=(s-this.globalData.compSize.h*(a/this.globalData.compSize.w))/2):(t=s/this.globalData.compSize.h,e=s/this.globalData.compSize.h,r=(a-this.globalData.compSize.w*(s/this.globalData.compSize.h))/2,i=0);var o=this.resizerElem.style;o.webkitTransform="matrix3d("+t+",0,0,0,0,"+e+",0,0,0,0,1,0,"+r+","+i+",0,1)",o.transform=o.webkitTransform},HybridRenderer.prototype.renderFrame=SVGRenderer.prototype.renderFrame,HybridRenderer.prototype.hide=function(){this.resizerElem.style.display="none"},HybridRenderer.prototype.show=function(){this.resizerElem.style.display="block"},HybridRenderer.prototype.initItems=function(){if(this.buildAllItems(),this.camera)this.camera.setup();else{var t,e=this.globalData.compSize.w,r=this.globalData.compSize.h,i=this.threeDElements.length;for(t=0;t1&&(s+=" C"+e.o[i-1][0]+","+e.o[i-1][1]+" "+e.i[0][0]+","+e.i[0][1]+" "+e.v[0][0]+","+e.v[0][1]),r.lastPath!==s){var n="";r.elem&&(e.c&&(n=t.inv?this.solidPath+s:s),r.elem.setAttribute("d",n)),r.lastPath=s}},MaskElement.prototype.destroy=function(){this.element=null,this.globalData=null,this.maskElement=null,this.data=null,this.masksProperties=null},HierarchyElement.prototype={initHierarchy:function(){this.hierarchy=[],this._isParent=!1,this.checkParenting()},setHierarchy:function(t){this.hierarchy=t},setAsParent:function(){this._isParent=!0},checkParenting:function(){void 0!==this.data.parent&&this.comp.buildElementParenting(this,this.data.parent,[])}},FrameElement.prototype={initFrame:function(){this._isFirstFrame=!1,this.dynamicProperties=[],this._mdf=!1},prepareProperties:function(t,e){var r,i=this.dynamicProperties.length;for(r=0;rt?!0!==this.isInRange&&(this.globalData._mdf=!0,this._mdf=!0,this.isInRange=!0,this.show()):!1!==this.isInRange&&(this.globalData._mdf=!0,this.isInRange=!1,this.hide())},renderRenderable:function(){var t,e=this.renderableComponents.length;for(t=0;t0;)h=i.transformers[u].mProps._mdf||h,m-=1,u-=1;if(h)for(m=g-i.styles[p].lvl,u=i.transformers.length-1;m>0;)d=i.transformers[u].mProps.v.props,f.transform(d[0],d[1],d[2],d[3],d[4],d[5],d[6],d[7],d[8],d[9],d[10],d[11],d[12],d[13],d[14],d[15]),m-=1,u-=1}else f=t;if(n=(c=i.sh.paths)._length,h){for(o="",s=0;s=1?v=.99:v<=-1&&(v=-.99);var b=o*v,_=Math.cos(g+e.a.v)*b+p[0],P=Math.sin(g+e.a.v)*b+p[1];h.setAttribute("fx",_),h.setAttribute("fy",P),l&&!e.g._collapsable&&(e.of.setAttribute("fx",_),e.of.setAttribute("fy",P))}}function o(t,e,r){var i=e.style,a=e.d;a&&(a._mdf||r)&&a.dashStr&&(i.pElem.setAttribute("stroke-dasharray",a.dashStr),i.pElem.setAttribute("stroke-dashoffset",a.dashoffset[0])),e.c&&(e.c._mdf||r)&&i.pElem.setAttribute("stroke","rgb("+bmFloor(e.c.v[0])+","+bmFloor(e.c.v[1])+","+bmFloor(e.c.v[2])+")"),(e.o._mdf||r)&&i.pElem.setAttribute("stroke-opacity",e.o.v),(e.w._mdf||r)&&(i.pElem.setAttribute("stroke-width",e.w.v),i.msElem&&i.msElem.setAttribute("stroke-width",e.w.v))}return{createRenderFunction:function(t){switch(t.ty){case"fl":return a;case"gf":return n;case"gs":return s;case"st":return o;case"sh":case"el":case"rc":case"sr":return i;case"tr":return r;default:return null}}}}();function ShapeTransformManager(){this.sequences={},this.sequenceList=[],this.transform_key_count=0}function CVShapeData(t,e,r,i){this.styledShapes=[],this.tr=[0,0,0,0,0,0];var a,s=4;"rc"===e.ty?s=5:"el"===e.ty?s=6:"sr"===e.ty&&(s=7),this.sh=ShapePropertyFactory.getShapeProp(t,e,s,t);var n,o=r.length;for(a=0;a=0;i-=1)r=t.transforms[i].transform.mProps.v.props,t.finalTransform.transform(r[0],r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9],r[10],r[11],r[12],r[13],r[14],r[15]);t._mdf=s},processSequences:function(t){var e,r=this.sequenceList.length;for(e=0;e=0&&!this.shapeModifiers[t].processShapes(this._isFirstFrame);t-=1);}},lcEnum:{1:"butt",2:"round",3:"square"},ljEnum:{1:"miter",2:"round",3:"bevel"},searchProcessedElement:function(t){for(var e=this.processedElements,r=0,i=e.length;r=0;r-=1)(this.completeLayers||this.elements[r])&&(this.elements[r].prepareFrame(this.renderedFrame-this.layers[r].st),this.elements[r]._mdf&&(this._mdf=!0))}},ICompElement.prototype.renderInnerContent=function(){var t,e=this.layers.length;for(t=0;t.1)&&this.audio.seek(this._currentTime/this.globalData.frameRate):(this.audio.play(),this.audio.seek(this._currentTime/this.globalData.frameRate),this._isPlaying=!0))},AudioElement.prototype.show=function(){},AudioElement.prototype.hide=function(){this.audio.pause(),this._isPlaying=!1},AudioElement.prototype.pause=function(){this.audio.pause(),this._isPlaying=!1,this._canPlay=!1},AudioElement.prototype.resume=function(){this._canPlay=!0},AudioElement.prototype.setRate=function(t){this.audio.rate(t)},AudioElement.prototype.volume=function(t){this.audio.volume(t)},AudioElement.prototype.getBaseElement=function(){return null},AudioElement.prototype.destroy=function(){},AudioElement.prototype.sourceRectAtTime=function(){},AudioElement.prototype.initExpressions=function(){},FootageElement.prototype.prepareFrame=function(){},extendPrototype([RenderableElement,BaseElement,FrameElement],FootageElement),FootageElement.prototype.getBaseElement=function(){return null},FootageElement.prototype.renderFrame=function(){},FootageElement.prototype.destroy=function(){},FootageElement.prototype.initExpressions=function(){this.layerInterface=FootageInterface(this)},FootageElement.prototype.getFootageData=function(){return this.footageData},extendPrototype([SVGRenderer,ICompElement,SVGBaseElement],SVGCompElement),extendPrototype([BaseElement,TransformElement,SVGBaseElement,HierarchyElement,FrameElement,RenderableDOMElement,ITextElement],SVGTextLottieElement),SVGTextLottieElement.prototype.createContent=function(){this.data.singleShape&&!this.globalData.fontManager.chars&&(this.textContainer=createNS("text"))},SVGTextLottieElement.prototype.buildTextContents=function(t){for(var e=0,r=t.length,i=[],a="";et?this.textSpans[t]:createNS(h?"path":"text"),b<=t&&(n.setAttribute("stroke-linecap","butt"),n.setAttribute("stroke-linejoin","round"),n.setAttribute("stroke-miterlimit","4"),this.textSpans[t]=n,this.layerElement.appendChild(n)),n.style.display="inherit"),p.reset(),p.scale(r.finalSize/100,r.finalSize/100),f&&(o[t].n&&(d=-y,m+=r.yOffset,m+=u?1:0,u=!1),this.applyTextPropertiesToMatrix(r,p,o[t].line,d,m),d+=o[t].l||0,d+=y),h?(l=(g=(v=this.globalData.fontManager.getCharData(r.finalText[t],i.fStyle,this.globalData.fontManager.getFontByName(r.f).fFamily))&&v.data||{}).shapes?g.shapes[0].it:[],f?c+=this.createPathShape(p,l):n.setAttribute("d",this.createPathShape(p,l))):(f&&n.setAttribute("transform","translate("+p.props[12]+","+p.props[13]+")"),n.textContent=o[t].val,n.setAttributeNS("http://www.w3.org/XML/1998/namespace","xml:space","preserve"));f&&n&&n.setAttribute("d",c)}else{var _=this.textContainer,P="start";switch(r.j){case 1:P="end";break;case 2:P="middle";break;default:P="start"}_.setAttribute("text-anchor",P),_.setAttribute("letter-spacing",y);var S=this.buildTextContents(r.finalText);for(e=S.length,m=r.ps?r.ps[1]+r.ascent:0,t=0;t1&&o&&this.setShapesAsAnimated(n)}},SVGShapeElement.prototype.setShapesAsAnimated=function(t){var e,r=t.length;for(e=0;e=0;o-=1){if((f=this.searchProcessedElement(t[o]))?e[o]=r[f-1]:t[o]._render=n,"fl"===t[o].ty||"st"===t[o].ty||"gf"===t[o].ty||"gs"===t[o].ty)f?e[o].style.closed=!1:e[o]=this.createStyleElement(t[o],a),t[o]._render&&i.appendChild(e[o].style.pElem),u.push(e[o].style);else if("gr"===t[o].ty){if(f)for(l=e[o].it.length,h=0;h=l?d<0?i:a:i+f*Math.pow((s-t)/d,1/r),p[c]=n,c+=1,o+=256/255;return p.join(" ")},SVGProLevelsFilter.prototype.renderFrame=function(t){if(t||this.filterManager._mdf){var e,r=this.filterManager.effectElements;this.feFuncRComposed&&(t||r[3].p._mdf||r[4].p._mdf||r[5].p._mdf||r[6].p._mdf||r[7].p._mdf)&&(e=this.getTableValue(r[3].p.v,r[4].p.v,r[5].p.v,r[6].p.v,r[7].p.v),this.feFuncRComposed.setAttribute("tableValues",e),this.feFuncGComposed.setAttribute("tableValues",e),this.feFuncBComposed.setAttribute("tableValues",e)),this.feFuncR&&(t||r[10].p._mdf||r[11].p._mdf||r[12].p._mdf||r[13].p._mdf||r[14].p._mdf)&&(e=this.getTableValue(r[10].p.v,r[11].p.v,r[12].p.v,r[13].p.v,r[14].p.v),this.feFuncR.setAttribute("tableValues",e)),this.feFuncG&&(t||r[17].p._mdf||r[18].p._mdf||r[19].p._mdf||r[20].p._mdf||r[21].p._mdf)&&(e=this.getTableValue(r[17].p.v,r[18].p.v,r[19].p.v,r[20].p.v,r[21].p.v),this.feFuncG.setAttribute("tableValues",e)),this.feFuncB&&(t||r[24].p._mdf||r[25].p._mdf||r[26].p._mdf||r[27].p._mdf||r[28].p._mdf)&&(e=this.getTableValue(r[24].p.v,r[25].p.v,r[26].p.v,r[27].p.v,r[28].p.v),this.feFuncB.setAttribute("tableValues",e)),this.feFuncA&&(t||r[31].p._mdf||r[32].p._mdf||r[33].p._mdf||r[34].p._mdf||r[35].p._mdf)&&(e=this.getTableValue(r[31].p.v,r[32].p.v,r[33].p.v,r[34].p.v,r[35].p.v),this.feFuncA.setAttribute("tableValues",e))}},SVGDropShadowEffect.prototype.renderFrame=function(t){if(t||this.filterManager._mdf){if((t||this.filterManager.effectElements[4].p._mdf)&&this.feGaussianBlur.setAttribute("stdDeviation",this.filterManager.effectElements[4].p.v/4),t||this.filterManager.effectElements[0].p._mdf){var e=this.filterManager.effectElements[0].p.v;this.feFlood.setAttribute("flood-color",rgbToHex(Math.round(255*e[0]),Math.round(255*e[1]),Math.round(255*e[2])))}if((t||this.filterManager.effectElements[1].p._mdf)&&this.feFlood.setAttribute("flood-opacity",this.filterManager.effectElements[1].p.v/255),t||this.filterManager.effectElements[2].p._mdf||this.filterManager.effectElements[3].p._mdf){var r=this.filterManager.effectElements[3].p.v,i=(this.filterManager.effectElements[2].p.v-90)*degToRads,a=r*Math.cos(i),s=r*Math.sin(i);this.feOffset.setAttribute("dx",a),this.feOffset.setAttribute("dy",s)}}};var _svgMatteSymbols=[];function SVGMatte3Effect(t,e,r){this.initialized=!1,this.filterManager=e,this.filterElem=t,this.elem=r,r.matteElement=createNS("g"),r.matteElement.appendChild(r.layerElement),r.matteElement.appendChild(r.transformedElement),r.baseElement=r.matteElement}function SVGEffects(t){var e,r,i=t.data.ef?t.data.ef.length:0,a=createElementID(),s=filtersFactory.createFilter(a,!0),n=0;for(this.filters=[],e=0;eo&&"xMidYMid slice"===h||n=0;t-=1)(this.completeLayers||this.elements[t])&&this.elements[t].renderFrame()},CVCompElement.prototype.destroy=function(){var t;for(t=this.layers.length-1;t>=0;t-=1)this.elements[t]&&this.elements[t].destroy();this.layers=null,this.elements=null},CVMaskElement.prototype.renderFrame=function(){if(this.hasMasks){var t,e,r,i,a=this.element.finalTransform.mat,s=this.element.canvasContext,n=this.masksProperties.length;for(s.beginPath(),t=0;t=0;s-=1){if((h=this.searchProcessedElement(t[s]))?e[s]=r[h-1]:t[s]._shouldRender=i,"fl"===t[s].ty||"st"===t[s].ty||"gf"===t[s].ty||"gs"===t[s].ty)h?e[s].style.closed=!1:e[s]=this.createStyleElement(t[s],m),f.push(e[s].style);else if("gr"===t[s].ty){if(h)for(o=e[s].it.length,n=0;n=0;a-=1)"tr"===e[a].ty?(s=r[a].transform,this.renderShapeTransform(t,s)):"sh"===e[a].ty||"el"===e[a].ty||"rc"===e[a].ty||"sr"===e[a].ty?this.renderPath(e[a],r[a]):"fl"===e[a].ty?this.renderFill(e[a],r[a],s):"st"===e[a].ty?this.renderStroke(e[a],r[a],s):"gf"===e[a].ty||"gs"===e[a].ty?this.renderGradientFill(e[a],r[a],s):"gr"===e[a].ty?this.renderShape(s,e[a].it,r[a].it):e[a].ty;i&&this.drawLayer()},CVShapeElement.prototype.renderStyledShape=function(t,e){if(this._isFirstFrame||e._mdf||t.transforms._mdf){var r,i,a,s=t.trNodes,n=e.paths,o=n._length;s.length=0;var h=t.transforms.finalTransform;for(a=0;a=1?c=.99:c<=-1&&(c=-.99);var f=l*c,d=Math.cos(p+e.a.v)*f+o[0],m=Math.sin(p+e.a.v)*f+o[1];i=n.createRadialGradient(d,m,0,o[0],o[1],l)}var u=t.g.p,y=e.g.c,g=1;for(s=0;s0&&o<1&&c[f].push(this.calculateF(o,t,e,r,i,f)):(h=s*s-4*n*a)>=0&&((l=(-s+bmSqrt(h))/(2*a))>0&&l<1&&c[f].push(this.calculateF(l,t,e,r,i,f)),(p=(-s-bmSqrt(h))/(2*a))>0&&p<1&&c[f].push(this.calculateF(p,t,e,r,i,f))));this.shapeBoundingBox.left=bmMin.apply(null,c[0]),this.shapeBoundingBox.top=bmMin.apply(null,c[1]),this.shapeBoundingBox.right=bmMax.apply(null,c[0]),this.shapeBoundingBox.bottom=bmMax.apply(null,c[1])},HShapeElement.prototype.calculateF=function(t,e,r,i,a,s){return bmPow(1-t,3)*e[s]+3*bmPow(1-t,2)*t*r[s]+3*(1-t)*bmPow(t,2)*i[s]+bmPow(t,3)*a[s]},HShapeElement.prototype.calculateBoundingBox=function(t,e){var r,i=t.length;for(r=0;r=t.x+t.width&&this.currentBBox.height+this.currentBBox.y>=t.y+t.height},HShapeElement.prototype.renderInnerContent=function(){if(this._renderShapeFrame(),!this.hidden&&(this._isFirstFrame||this._mdf)){var t=this.tempBoundingBox,e=999999;if(t.x=e,t.xMax=-e,t.y=e,t.yMax=-e,this.calculateBoundingBox(this.itemsData,t),t.width=t.xMax=0;t-=1){var i=this.hierarchy[t].finalTransform.mProp;this.mat.translate(-i.p.v[0],-i.p.v[1],i.p.v[2]),this.mat.rotateX(-i.or.v[0]).rotateY(-i.or.v[1]).rotateZ(i.or.v[2]),this.mat.rotateX(-i.rx.v).rotateY(-i.ry.v).rotateZ(i.rz.v),this.mat.scale(1/i.s.v[0],1/i.s.v[1],1/i.s.v[2]),this.mat.translate(i.a.v[0],i.a.v[1],i.a.v[2])}if(this.p?this.mat.translate(-this.p.v[0],-this.p.v[1],this.p.v[2]):this.mat.translate(-this.px.v,-this.py.v,this.pz.v),this.a){var a;a=this.p?[this.p.v[0]-this.a.v[0],this.p.v[1]-this.a.v[1],this.p.v[2]-this.a.v[2]]:[this.px.v-this.a.v[0],this.py.v-this.a.v[1],this.pz.v-this.a.v[2]];var s=Math.sqrt(Math.pow(a[0],2)+Math.pow(a[1],2)+Math.pow(a[2],2)),n=[a[0]/s,a[1]/s,a[2]/s],o=Math.sqrt(n[2]*n[2]+n[0]*n[0]),h=Math.atan2(n[1],o),l=Math.atan2(n[0],-n[2]);this.mat.rotateY(l).rotateX(-h)}this.mat.rotateX(-this.rx.v).rotateY(-this.ry.v).rotateZ(this.rz.v),this.mat.rotateX(-this.or.v[0]).rotateY(-this.or.v[1]).rotateZ(this.or.v[2]),this.mat.translate(this.globalData.compSize.w/2,this.globalData.compSize.h/2,0),this.mat.translate(0,0,this.pe.v);var p=!this._prevMat.equals(this.mat);if((p||this.pe._mdf)&&this.comp.threeDElements){var c,f,d;for(e=this.comp.threeDElements.length,t=0;t=0;r-=1)e[r].animation.destroy(t)},t.freeze=function(){n=!0},t.unfreeze=function(){n=!1,m()},t.setVolume=function(t,r){var a;for(a=0;athis.animationData.op&&(this.animationData.op=t.op,this.totalFrames=Math.floor(t.op-this.animationData.ip));var e,r,i=this.animationData.layers,a=i.length,s=t.layers,n=s.length;for(r=0;rthis.timeCompleted&&(this.currentFrame=this.timeCompleted),this.trigger("enterFrame"),this.renderFrame()},AnimationItem.prototype.renderFrame=function(){if(!1!==this.isLoaded&&this.renderer)try{this.renderer.renderFrame(this.currentFrame+this.firstFrame)}catch(t){this.triggerRenderFrameError(t)}},AnimationItem.prototype.play=function(t){t&&this.name!==t||!0===this.isPaused&&(this.isPaused=!1,this.audioController.resume(),this._idle&&(this._idle=!1,this.trigger("_active")))},AnimationItem.prototype.pause=function(t){t&&this.name!==t||!1===this.isPaused&&(this.isPaused=!0,this._idle=!0,this.trigger("_idle"),this.audioController.pause())},AnimationItem.prototype.togglePause=function(t){t&&this.name!==t||(!0===this.isPaused?this.play():this.pause())},AnimationItem.prototype.stop=function(t){t&&this.name!==t||(this.pause(),this.playCount=0,this._completedLoop=!1,this.setCurrentRawFrameValue(0))},AnimationItem.prototype.getMarkerData=function(t){for(var e,r=0;r=this.totalFrames-1&&this.frameModifier>0?this.loop&&this.playCount!==this.loop?e>=this.totalFrames?(this.playCount+=1,this.checkSegments(e%this.totalFrames)||(this.setCurrentRawFrameValue(e%this.totalFrames),this._completedLoop=!0,this.trigger("loopComplete"))):this.setCurrentRawFrameValue(e):this.checkSegments(e>this.totalFrames?e%this.totalFrames:0)||(r=!0,e=this.totalFrames-1):e<0?this.checkSegments(e%this.totalFrames)||(!this.loop||this.playCount--<=0&&!0!==this.loop?(r=!0,e=0):(this.setCurrentRawFrameValue(this.totalFrames+e%this.totalFrames),this._completedLoop?this.trigger("loopComplete"):this._completedLoop=!0)):this.setCurrentRawFrameValue(e),r&&(this.setCurrentRawFrameValue(e),this.pause(),this.trigger("complete"))}},AnimationItem.prototype.adjustSegment=function(t,e){this.playCount=0,t[1]0&&(this.playSpeed<0?this.setSpeed(-this.playSpeed):this.setDirection(-1)),this.totalFrames=t[0]-t[1],this.timeCompleted=this.totalFrames,this.firstFrame=t[1],this.setCurrentRawFrameValue(this.totalFrames-.001-e)):t[1]>t[0]&&(this.frameModifier<0&&(this.playSpeed<0?this.setSpeed(-this.playSpeed):this.setDirection(1)),this.totalFrames=t[1]-t[0],this.timeCompleted=this.totalFrames,this.firstFrame=t[0],this.setCurrentRawFrameValue(.001+e)),this.trigger("segmentStart")},AnimationItem.prototype.setSegment=function(t,e){var r=-1;this.isPaused&&(this.currentRawFrame+this.firstFramee&&(r=e-t)),this.firstFrame=t,this.totalFrames=e-t,this.timeCompleted=this.totalFrames,-1!==r&&this.goToAndStop(r,!0)},AnimationItem.prototype.playSegments=function(t,e){if(e&&(this.segments.length=0),"object"==typeof t[0]){var r,i=t.length;for(r=0;rr){var i=r;r=e,e=i}return Math.min(Math.max(t,e),r)}function radiansToDegrees(t){return t/degToRads}var radians_to_degrees=radiansToDegrees;function degreesToRadians(t){return t*degToRads}var degrees_to_radians=radiansToDegrees,helperLengthArray=[0,0,0,0,0,0];function length(t,e){if("number"==typeof t||t instanceof Number)return e=e||0,Math.abs(t-e);var r;e||(e=helperLengthArray);var i=Math.min(t.length,e.length),a=0;for(r=0;r.5?l/(2-n-o):l/(n+o),n){case i:e=(a-s)/l+(a1&&(r-=1),r<1/6?t+6*(e-t)*r:r<.5?e:r<2/3?t+(e-t)*(2/3-r)*6:t}function hslToRgb(t){var e,r,i,a=t[0],s=t[1],n=t[2];if(0===s)e=n,i=n,r=n;else{var o=n<.5?n*(1+s):n+s-n*s,h=2*n-o;e=hue2rgb(h,o,a+1/3),r=hue2rgb(h,o,a),i=hue2rgb(h,o,a-1/3)}return[e,r,i,t[3]]}function linear(t,e,r,i,a){if(void 0!==i&&void 0!==a||(i=e,a=r,e=0,r=1),r=r)return a;var n,o=r===e?0:(t-e)/(r-e);if(!i.length)return i+(a-i)*o;var h=i.length,l=createTypedArray("float32",h);for(n=0;n1){for(i=0;i1?e=1:e<0&&(e=0);var n=t(e);if($bm_isInstanceOfArray(a)){var o,h=a.length,l=createTypedArray("float32",h);for(o=0;odata.k[e].t&&tdata.k[e+1].t-t?(r=e+2,i=data.k[e+1].t):(r=e+1,i=data.k[e].t);break}}-1===r&&(r=e+1,i=data.k[e].t)}else r=0,i=0;var s={};return s.index=r,s.time=i/elem.comp.globalData.frameRate,s}function key(t){var e,r,i;if(!data.k.length||"number"==typeof data.k[0])throw new Error("The property has no keyframe at index "+t);t-=1,e={time:data.k[t].t/elem.comp.globalData.frameRate,value:[]};var a=Object.prototype.hasOwnProperty.call(data.k[t],"s")?data.k[t].s:data.k[t-1].e;for(i=a.length,r=0;rl.length-1)&&(e=l.length-1),i=p-(a=l[l.length-1-e].t)),"pingpong"===t){if(Math.floor((h-a)/i)%2!=0)return this.getValueAtTime((i-(h-a)%i+a)/this.comp.globalData.frameRate,0)}else{if("offset"===t){var c=this.getValueAtTime(a/this.comp.globalData.frameRate,0),f=this.getValueAtTime(p/this.comp.globalData.frameRate,0),d=this.getValueAtTime(((h-a)%i+a)/this.comp.globalData.frameRate,0),m=Math.floor((h-a)/i);if(this.pv.length){for(n=(o=new Array(c.length)).length,s=0;s=p)return this.pv;if(r?a=p+(i=e?Math.abs(this.elem.comp.globalData.frameRate*e):Math.max(0,this.elem.data.op-p)):((!e||e>l.length-1)&&(e=l.length-1),i=(a=l[e].t)-p),"pingpong"===t){if(Math.floor((p-h)/i)%2==0)return this.getValueAtTime(((p-h)%i+p)/this.comp.globalData.frameRate,0)}else{if("offset"===t){var c=this.getValueAtTime(p/this.comp.globalData.frameRate,0),f=this.getValueAtTime(a/this.comp.globalData.frameRate,0),d=this.getValueAtTime((i-(p-h)%i+p)/this.comp.globalData.frameRate,0),m=Math.floor((p-h)/i)+1;if(this.pv.length){for(n=(o=new Array(c.length)).length,s=0;s1?(a+t-s)/(e-1):1,o=0,h=0;for(r=this.pv.length?createTypedArray("float32",this.pv.length):0;on){var p=o,c=r.c&&o===h-1?0:o+1,f=(n-l)/s[o].addedLength;i=bez.getPointInSegment(r.v[p],r.v[c],r.o[p],r.i[c],f,s[o]);break}l+=s[o].addedLength,o+=1}return i||(i=r.c?[r.v[0][0],r.v[0][1]]:[r.v[r._length-1][0],r.v[r._length-1][1]]),i},vectorOnPath:function(t,e,r){1==t?t=this.v.c:0==t&&(t=.999);var i=this.pointOnPath(t,e),a=this.pointOnPath(t+.001,e),s=a[0]-i[0],n=a[1]-i[1],o=Math.sqrt(Math.pow(s,2)+Math.pow(n,2));return 0===o?[0,0]:"tangent"===r?[s/o,n/o]:[-n/o,s/o]},tangentOnPath:function(t,e){return this.vectorOnPath(t,e,"tangent")},normalOnPath:function(t,e){return this.vectorOnPath(t,e,"normal")},setGroupProperty:expressionHelpers.setGroupProperty,getValueAtTime:expressionHelpers.getStaticValueAtTime},extendPrototype([l],o),extendPrototype([l],h),h.prototype.getValueAtTime=function(t){return this._cachingAtTime||(this._cachingAtTime={shapeValue:shapePool.clone(this.pv),lastIndex:0,lastTime:initialDefaultFrame}),t*=this.elem.globalData.frameRate,(t-=this.offsetTime)!==this._cachingAtTime.lastTime&&(this._cachingAtTime.lastIndex=this._cachingAtTime.lastTime1&&(defaultCurveSegments=t);roundValues(!(defaultCurveSegments>=50))}function inBrowser(){return"undefined"!=typeof navigator}function installPlugin(t,e){"expressions"===t&&(expressionsPlugin=e)}function getFactory(t){switch(t){case"propertyFactory":return PropertyFactory;case"shapePropertyFactory":return ShapePropertyFactory;case"matrix":return Matrix;default:return null}}function checkReady(){"complete"===document.readyState&&(clearInterval(readyStateCheckInterval),searchAnimations())}function getQueryVariable(t){for(var e=queryString.split("&"),r=0;rObject.prototype.hasOwnProperty.call(t,e))}function fromURL(t){return _fromURL.apply(this,arguments)}function _fromURL(){return(_fromURL=_asyncToGenerator((function*(t){if("string"!=typeof t)throw new Error("The url value must be a string");var e;try{var r=new URL(t),i=yield fetch(r.toString());e=yield i.json()}catch(t){throw new Error("An error occurred while trying to load the Lottie file from URL")}return e}))).apply(this,arguments)}exports.PlayerState=void 0,PlayerState=exports.PlayerState||(exports.PlayerState={}),PlayerState.Destroyed="destroyed",PlayerState.Error="error",PlayerState.Frozen="frozen",PlayerState.Loading="loading",PlayerState.Paused="paused",PlayerState.Playing="playing",PlayerState.Stopped="stopped",exports.PlayMode=void 0,PlayMode=exports.PlayMode||(exports.PlayMode={}),PlayMode.Bounce="bounce",PlayMode.Normal="normal",exports.PlayerEvents=void 0,PlayerEvents=exports.PlayerEvents||(exports.PlayerEvents={}),PlayerEvents.Complete="complete",PlayerEvents.Destroyed="destroyed",PlayerEvents.Error="error",PlayerEvents.Frame="frame",PlayerEvents.Freeze="freeze",PlayerEvents.Load="load",PlayerEvents.Loop="loop",PlayerEvents.Pause="pause",PlayerEvents.Play="play",PlayerEvents.Ready="ready",PlayerEvents.Rendered="rendered",PlayerEvents.Stop="stop",exports.LottiePlayer=class extends LitElement{constructor(){super(...arguments),this.autoplay=!1,this.background="transparent",this.controls=!1,this.currentState=exports.PlayerState.Loading,this.description="Lottie animation",this.direction=1,this.hover=!1,this.intermission=1,this.loop=!1,this.mode=exports.PlayMode.Normal,this.preserveAspectRatio="xMidYMid meet",this.renderer="svg",this.speed=1,this._io=void 0,this._counter=0}load(t){var e=this;return _asyncToGenerator((function*(){if(e.shadowRoot){var r={container:e.container,loop:!1,autoplay:!1,renderer:e.renderer,rendererSettings:{preserveAspectRatio:e.preserveAspectRatio,clearCanvas:!1,progressiveLoad:!0,hideOnTransparent:!0}};try{var i=parseSrc(t),a={},s="string"==typeof i?"path":"animationData";e._lottie&&e._lottie.destroy(),e._lottie=lottie.loadAnimation(Object.assign(Object.assign({},r),{[s]:i})),e._attachEventListeners(),"path"===s?(a=yield fromURL(i),s="animationData"):a=i,isLottie(a)||(e.currentState=exports.PlayerState.Error,e.dispatchEvent(new CustomEvent(exports.PlayerEvents.Error)))}catch(t){return e.currentState=exports.PlayerState.Error,void e.dispatchEvent(new CustomEvent(exports.PlayerEvents.Error))}}}))()}getLottie(){return this._lottie}play(){this._lottie&&(this._lottie.play(),this.currentState=exports.PlayerState.Playing,this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Play)))}pause(){this._lottie&&(this._lottie.pause(),this.currentState=exports.PlayerState.Paused,this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Pause)))}stop(){this._lottie&&(this._counter=0,this._lottie.stop(),this.currentState=exports.PlayerState.Stopped,this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Stop)))}destroy(){this._lottie&&(this._lottie.destroy(),this.currentState=exports.PlayerState.Destroyed,this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Destroyed)),this.remove())}seek(t){if(this._lottie){var e=/^(\d+)(%?)$/.exec(t.toString());if(e){var r="%"===e[2]?this._lottie.totalFrames*Number(e[1])/100:Number(e[1]);this.seeker=r,this.currentState===exports.PlayerState.Playing?this._lottie.goToAndPlay(r,!0):(this._lottie.goToAndStop(r,!0),this._lottie.pause())}}}snapshot(){var t=!(arguments.length>0&&void 0!==arguments[0])||arguments[0];if(this.shadowRoot){var e=this.shadowRoot.querySelector(".animation svg"),r=(new XMLSerializer).serializeToString(e);if(t){var i=document.createElement("a");i.href="data:image/svg+xml;charset=utf-8,".concat(encodeURIComponent(r)),i.download="download_".concat(this.seeker,".svg"),document.body.appendChild(i),i.click(),document.body.removeChild(i)}return r}}setSpeed(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;this._lottie&&this._lottie.setSpeed(t)}setDirection(t){this._lottie&&this._lottie.setDirection(t)}setLooping(t){this._lottie&&(this.loop=t,this._lottie.loop=t)}togglePlay(){return this.currentState===exports.PlayerState.Playing?this.pause():this.play()}toggleLooping(){this.setLooping(!this.loop)}resize(){this._lottie&&this._lottie.resize()}static get styles(){return styles}disconnectedCallback(){this._io&&(this._io.disconnect(),this._io=void 0),document.removeEventListener("visibilitychange",()=>this._onVisibilityChange()),this.destroy()}render(){var t=this.controls?"main controls":"main",e=this.controls?"animation controls":"animation";return html(_templateObject||(_templateObject=_taggedTemplateLiteral([' \n \n ',"\n \n ","\n "])),t,this.description,e,this.background,this.currentState===exports.PlayerState.Error?html(_templateObject2||(_templateObject2=_taggedTemplateLiteral(['
⚠️
']))):void 0,this.controls?this.renderControls():void 0)}firstUpdated(){"IntersectionObserver"in window&&(this._io=new IntersectionObserver(t=>{t[0].isIntersecting?this.currentState===exports.PlayerState.Frozen&&this.play():this.currentState===exports.PlayerState.Playing&&this.freeze()}),this._io.observe(this.container)),void 0!==document.hidden&&document.addEventListener("visibilitychange",()=>this._onVisibilityChange()),this.src&&this.load(this.src),this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Rendered))}renderControls(){var t=this.currentState===exports.PlayerState.Playing,e=this.currentState===exports.PlayerState.Paused,r=this.currentState===exports.PlayerState.Stopped;return html(_templateObject3||(_templateObject3=_taggedTemplateLiteral(['\n \n \n ','\n \n \n \n \n \n \n \n \n \n \n \n '])),this.togglePlay,t||e?"active":"",html(t?_templateObject4||(_templateObject4=_taggedTemplateLiteral(['\n \n '])):_templateObject5||(_templateObject5=_taggedTemplateLiteral(['\n \n ']))),this.stop,r?"active":"",this.seeker,this._handleSeekChange,()=>{this._prevState=this.currentState,this.freeze()},()=>{this._prevState===exports.PlayerState.Playing&&this.play()},this.seeker,this.toggleLooping,this.loop?"active":"")}_onVisibilityChange(){!0===document.hidden&&this.currentState===exports.PlayerState.Playing?this.freeze():this.currentState===exports.PlayerState.Frozen&&this.play()}_handleSeekChange(t){if(this._lottie&&!isNaN(t.target.value)){var e=t.target.value/100*this._lottie.totalFrames;this.seek(e)}}_attachEventListeners(){this._lottie.addEventListener("enterFrame",()=>{this.seeker=this._lottie.currentFrame/this._lottie.totalFrames*100,this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Frame,{detail:{frame:this._lottie.currentFrame,seeker:this.seeker}}))}),this._lottie.addEventListener("complete",()=>{this.currentState===exports.PlayerState.Playing?!this.loop||this.count&&this._counter>=this.count?this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Complete)):this.mode===exports.PlayMode.Bounce?(this.count&&(this._counter+=.5),setTimeout(()=>{this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Loop)),this.currentState===exports.PlayerState.Playing&&(this._lottie.setDirection(-1*this._lottie.playDirection),this._lottie.play())},this.intermission)):(this.count&&(this._counter+=1),window.setTimeout(()=>{this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Loop)),this.currentState===exports.PlayerState.Playing&&(this._lottie.stop(),this._lottie.play())},this.intermission)):this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Complete))}),this._lottie.addEventListener("DOMLoaded",()=>{this.setSpeed(this.speed),this.setDirection(this.direction),this.autoplay&&this.play(),this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Ready))}),this._lottie.addEventListener("data_ready",()=>{this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Load))}),this._lottie.addEventListener("data_failed",()=>{this.currentState=exports.PlayerState.Error,this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Error))}),this.container.addEventListener("mouseenter",()=>{this.hover&&this.currentState!==exports.PlayerState.Playing&&this.play()}),this.container.addEventListener("mouseleave",()=>{this.hover&&this.currentState===exports.PlayerState.Playing&&this.stop()})}freeze(){this._lottie&&(this._lottie.pause(),this.currentState=exports.PlayerState.Frozen,this.dispatchEvent(new CustomEvent(exports.PlayerEvents.Freeze)))}},__decorate([property({type:Boolean})],exports.LottiePlayer.prototype,"autoplay",void 0),__decorate([property({type:String,reflect:!0})],exports.LottiePlayer.prototype,"background",void 0),__decorate([property({type:Boolean})],exports.LottiePlayer.prototype,"controls",void 0),__decorate([property({type:Number})],exports.LottiePlayer.prototype,"count",void 0),__decorate([property({type:String})],exports.LottiePlayer.prototype,"currentState",void 0),__decorate([property({type:String})],exports.LottiePlayer.prototype,"description",void 0),__decorate([property({type:Number})],exports.LottiePlayer.prototype,"direction",void 0),__decorate([property({type:Boolean})],exports.LottiePlayer.prototype,"hover",void 0),__decorate([property()],exports.LottiePlayer.prototype,"intermission",void 0),__decorate([property({type:Boolean,reflect:!0})],exports.LottiePlayer.prototype,"loop",void 0),__decorate([property()],exports.LottiePlayer.prototype,"mode",void 0),__decorate([property({type:String})],exports.LottiePlayer.prototype,"preserveAspectRatio",void 0),__decorate([property({type:String})],exports.LottiePlayer.prototype,"renderer",void 0),__decorate([property()],exports.LottiePlayer.prototype,"seeker",void 0),__decorate([property({type:Number})],exports.LottiePlayer.prototype,"speed",void 0),__decorate([property({type:String})],exports.LottiePlayer.prototype,"src",void 0),__decorate([query(".animation")],exports.LottiePlayer.prototype,"container",void 0),exports.LottiePlayer=__decorate([customElement("lottie-player")],exports.LottiePlayer),exports.parseSrc=parseSrc,Object.defineProperty(exports,"__esModule",{value:!0})})); +//# sourceMappingURL=lottie-player.js.map diff --git a/doc/mkdocs/js/termynal.js b/doc/mkdocs/js/termynal.js new file mode 100644 index 000000000..147bf4e66 --- /dev/null +++ b/doc/mkdocs/js/termynal.js @@ -0,0 +1,265 @@ +/** + * termynal.js + * A lightweight, modern and extensible animated terminal window, using + * async/await. + * + * @author Ines Montani + * @version 0.0.1 + * @license MIT + */ + +'use strict'; + +/** Generate a terminal widget. */ +class Termynal { + /** + * Construct the widget's settings. + * @param {(string|Node)=} container - Query selector or container element. + * @param {Object=} options - Custom settings. + * @param {string} options.prefix - Prefix to use for data attributes. + * @param {number} options.startDelay - Delay before animation, in ms. + * @param {number} options.typeDelay - Delay between each typed character, in ms. + * @param {number} options.lineDelay - Delay between each line, in ms. + * @param {number} options.progressLength - Number of characters displayed as progress bar. + * @param {string} options.progressChar – Character to use for progress bar, defaults to █. + * @param {number} options.progressPercent - Max percent of progress. + * @param {string} options.cursor – Character to use for cursor, defaults to ▋. + * @param {Object[]} lineData - Dynamically loaded line data objects. + * @param {boolean} options.noInit - Don't initialise the animation. + */ + constructor(container = '#termynal', options = {}) { + this.container = (typeof container === 'string') ? document.querySelector(container) : container; + this.pfx = `data-${options.prefix || 'ty'}`; + this.originalStartDelay = this.startDelay = options.startDelay + || parseFloat(this.container.getAttribute(`${this.pfx}-startDelay`)) || 300; + this.originalTypeDelay = this.typeDelay = options.typeDelay + || parseFloat(this.container.getAttribute(`${this.pfx}-typeDelay`)) || 30; + this.originalLineDelay = this.lineDelay = options.lineDelay + || parseFloat(this.container.getAttribute(`${this.pfx}-lineDelay`)) || 1500; + this.progressLength = options.progressLength + || parseFloat(this.container.getAttribute(`${this.pfx}-progressLength`)) || 40; + this.progressChar = options.progressChar + || this.container.getAttribute(`${this.pfx}-progressChar`) || '█'; + this.progressPercent = options.progressPercent + || parseFloat(this.container.getAttribute(`${this.pfx}-progressPercent`)) || 100; + this.cursor = options.cursor + || this.container.getAttribute(`${this.pfx}-cursor`) || '▋'; + this.lineData = this.lineDataToElements(options.lineData || []); + this.loadLines() + if (!options.noInit) this.init() + } + + loadLines() { + // Load all the lines and create the container so that the size is fixed + // Otherwise it would be changing and the user viewport would be constantly + // moving as she/he scrolls + const finish = this.generateFinish() + finish.style.visibility = 'hidden' + this.container.appendChild(finish) + // Appends dynamically loaded lines to existing line elements. + this.lines = [...this.container.querySelectorAll(`[${this.pfx}]`)].concat(this.lineData); + for (let line of this.lines) { + line.style.visibility = 'hidden' + this.container.appendChild(line) + } + const restart = this.generateRestart() + restart.style.visibility = 'hidden' + this.container.appendChild(restart) + this.container.setAttribute('data-termynal', ''); + } + + /** + * Initialise the widget, get lines, clear container and start animation. + */ + init() { + /** + * Calculates width and height of Termynal container. + * If container is empty and lines are dynamically loaded, defaults to browser `auto` or CSS. + */ + const containerStyle = getComputedStyle(this.container); + this.container.style.width = containerStyle.width !== '0px' ? + containerStyle.width : undefined; + this.container.style.minHeight = containerStyle.height !== '0px' ? + containerStyle.height : undefined; + + this.container.setAttribute('data-termynal', ''); + this.container.innerHTML = ''; + for (let line of this.lines) { + line.style.visibility = 'visible' + } + this.start(); + } + + /** + * Start the animation and rener the lines depending on their data attributes. + */ + async start() { + this.addFinish() + await this._wait(this.startDelay); + + for (let line of this.lines) { + const type = line.getAttribute(this.pfx); + const delay = line.getAttribute(`${this.pfx}-delay`) || this.lineDelay; + + if (type == 'input') { + line.setAttribute(`${this.pfx}-cursor`, this.cursor); + await this.type(line); + await this._wait(delay); + } + + else if (type == 'progress') { + await this.progress(line); + await this._wait(delay); + } + + else { + this.container.appendChild(line); + await this._wait(delay); + } + + line.removeAttribute(`${this.pfx}-cursor`); + } + this.addRestart() + this.finishElement.style.visibility = 'hidden' + this.lineDelay = this.originalLineDelay + this.typeDelay = this.originalTypeDelay + this.startDelay = this.originalStartDelay + } + + generateRestart() { + const restart = document.createElement('a') + restart.onclick = (e) => { + e.preventDefault() + this.container.innerHTML = '' + this.init() + } + restart.href = '#' + restart.setAttribute('data-terminal-control', '') + restart.innerHTML = "restart ↻" + return restart + } + + generateFinish() { + const finish = document.createElement('a') + finish.onclick = (e) => { + e.preventDefault() + this.lineDelay = 0 + this.typeDelay = 0 + this.startDelay = 0 + } + finish.href = '#' + finish.setAttribute('data-terminal-control', '') + finish.innerHTML = "fast →" + this.finishElement = finish + return finish + } + + addRestart() { + const restart = this.generateRestart() + this.container.appendChild(restart) + } + + addFinish() { + const finish = this.generateFinish() + this.container.appendChild(finish) + } + + /** + * Animate a typed line. + * @param {Node} line - The line element to render. + */ + async type(line) { + const chars = [...line.textContent]; + line.textContent = ''; + this.container.appendChild(line); + + for (let char of chars) { + const delay = line.getAttribute(`${this.pfx}-typeDelay`) || this.typeDelay; + await this._wait(delay); + line.textContent += char; + } + } + + /** + * Animate a progress bar. + * @param {Node} line - The line element to render. + */ + async progress(line) { + const progressLength = line.getAttribute(`${this.pfx}-progressLength`) + || this.progressLength; + const progressChar = line.getAttribute(`${this.pfx}-progressChar`) + || this.progressChar; + const chars = progressChar.repeat(progressLength); + const progressPercent = line.getAttribute(`${this.pfx}-progressPercent`) + || this.progressPercent; + line.textContent = ''; + this.container.appendChild(line); + + for (let i = 1; i < chars.length + 1; i++) { + await this._wait(this.typeDelay); + const percent = Math.round(i / chars.length * 100); + line.textContent = `${chars.slice(0, i)} ${percent}%`; + if (percent>progressPercent) { + break; + } + } + } + + /** + * Helper function for animation delays, called with `await`. + * @param {number} time - Timeout, in ms. + */ + _wait(time) { + return new Promise(resolve => setTimeout(resolve, time)); + } + + /** + * Converts line data objects into line elements. + * + * @param {Object[]} lineData - Dynamically loaded lines. + * @param {Object} line - Line data object. + * @returns {Element[]} - Array of line elements. + */ + lineDataToElements(lineData) { + return lineData.map(line => { + let div = document.createElement('div'); + div.innerHTML = `${line.value || ''}`; + + return div.firstElementChild; + }); + } + + /** + * Helper function for generating attributes string. + * + * @param {Object} line - Line data object. + * @returns {string} - String of attributes. + */ + _attributes(line) { + let attrs = ''; + for (let prop in line) { + // Custom add class + if (prop === 'class') { + attrs += ` class=${line[prop]} ` + continue + } + if (prop === 'type') { + attrs += `${this.pfx}="${line[prop]}" ` + } else if (prop !== 'value') { + attrs += `${this.pfx}-${prop}="${line[prop]}" ` + } + } + + return attrs; + } +} + +/** +* HTML API: If current script has container(s) specified, initialise Termynal. +*/ +if (document.currentScript.hasAttribute('data-termynal-container')) { + const containers = document.currentScript.getAttribute('data-termynal-container'); + containers.split('|') + .forEach(container => new Termynal(container)) +} + diff --git a/doc/mkdocs/requirements.txt b/doc/mkdocs/requirements.txt new file mode 100644 index 000000000..e287604b2 --- /dev/null +++ b/doc/mkdocs/requirements.txt @@ -0,0 +1,7 @@ +mike +mkdocs-material +python-markdown-math +mkdocs-awesome-pages-plugin +mkdocs-render-swagger-plugin +mkdocs-static-i18n +git+https://github.com/jarviszeng-zjc/markdown-include-snippet.git@develop-0.1.0#egg=markdown-include-snippet diff --git a/doc/mkdocs/theme/README.md b/doc/mkdocs/theme/README.md new file mode 100644 index 000000000..d5ce5dcea --- /dev/null +++ b/doc/mkdocs/theme/README.md @@ -0,0 +1 @@ +Mostly copied from https://github.com/cirruslabs/cirrus-ci-docs/tree/master/theme diff --git a/doc/mkdocs/theme/overrides/home.html b/doc/mkdocs/theme/overrides/home.html new file mode 100644 index 000000000..a43f46c49 --- /dev/null +++ b/doc/mkdocs/theme/overrides/home.html @@ -0,0 +1,92 @@ +{% extends "base.html" %} + + +{% block tabs %} +{{ super() }} + + + + + +
+
+
+ +
+ + + + +
+ + +
+

{{page.title}}

+

FATE Flow Base on: +

    +
  • Shared-State Scheduling Architecture
    • +
  • Secure Multi-Party Communication
    • +
+

+

Providing production-level service capabilities: +

    +
  • Data Access
    • +
  • Component Registry
    • +
  • Federated Job&Task Scheduling
    • +
  • Multi-Party Resource Coordination
    • +
  • Data Flow Tracking
    • +
  • Real-Time Monitoring
    • +
  • Federated Model Registry
    • +
  • Multi-Party Cooperation Authority Management
    • +
  • CLI, REST API, Python API
    • +
+

+ Learn More + GitHub + +
+
+
+
+{% endblock %} + + +{% block content %}{% endblock %} + + +{% block footer %}{% endblock %} diff --git a/doc/mkdocs/theme/overrides/home.zh.html b/doc/mkdocs/theme/overrides/home.zh.html new file mode 100644 index 000000000..b44cf65e0 --- /dev/null +++ b/doc/mkdocs/theme/overrides/home.zh.html @@ -0,0 +1,90 @@ +{% extends "base.html" %} + + +{% block tabs %} +{{ super() }} + + + + + +
+
+
+ +
+ + + + +
+ + +
+

{{page.title}}

+

FATE Flow 基于 +

    +
  • 共享状态调度架构
    • +
  • 多方安全通信
    • +
+

+

实现了端到端的联邦学习作业调度生产级服务，支持 +

    +
  • 数据接入
    • +
  • 任务组件注册中心
    • +
  • 联合作业&任务调度
    • +
  • 多方资源协调
    • +
  • 数据流动追踪
    • +
  • 作业实时监测
    • +
  • 联合模型注册中心
    • +
  • 多方合作权限管理
    • +
+

+ Learn More + GitHub +
+
+
+
+{% endblock %} + + +{% block content %}{% endblock %} + + +{% block footer %}{% endblock %} diff --git a/doc/quick_start.md b/doc/quick_start.md index 736a1dc70..bfd79833c 100644 --- a/doc/quick_start.md +++ b/doc/quick_start.md @@ -1,222 +1,332 @@ -### 1. FATE Flow 2.0.0-alpha 部署 - -#### 1.1 源码获取 -##### 1.1.1 从github拉取源码 - - [FATE](https://github.com/FederatedAI/FATE/tree/release-2.0-alpha) - - [FATE-Flow](https://github.com/FederatedAI/FATE-Flow/tree/release-2.0-alpha) -##### 1.1.2 新建部署目录: -```shell -mkdir -p /data/projects/fate2.0 -``` -##### 1.1.3 将源码放到部署目录 -```shell - # fate flow - mv ./FATE-Flow /data/projects/fate2.0/fate_flow - # fate算法包 - mv ./FATE/python /data/projects/fate2.0/python -``` -#### 1.2 依赖 -##### 1.2.1 miniconda安装 -```shell -wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/resources/Miniconda3-py38_4.12.0-Linux-x86_64.sh -#创建python虚拟化安装目录 -mkdir -p /data/projects/fate2.0/common/python/venv - -#安装miniconda3 -bash Miniconda3-py38_4.12.0-Linux-x86_64.sh -b -p /data/projects/fate2.0/common/miniconda3 -#创建虚拟化环境 -/data/projects/fate2.0/common/miniconda3/bin/python3.8 -m venv /data/projects/fate2.0/common/python/venv -``` - -##### 1.2.2 依赖安装 -```shell -source /data/projects/fate2.0/common/python/venv/bin/activate -cd /data/projects/fate2.0/fate_flow/python -pip install -r requirements.txt -``` -详细依赖参考： [requirements.txt](../python/requirements.txt) - -#### 1.3 修改配置 -#### 1.3.1 配置说明 -- 系统配置文件[service_conf.yaml](../conf/service_conf.yaml)说明： -```yaml -force_use_sqlite: 是否强制使用sqlite作为数据库 -party_id: 站点id -fateflow: - host: 服务ip - http_port: http端口 - grpc_port: grpc端口 - proxy_name: 命令通道服务名，支持rollsite/nginx/osx, 需要在下面的federation中配置具体的地址 -database: 数据库连接信息，若未部署mysql，可将force_use_sqlite设置为true -default_engines: - computing: 计算引擎, 可填:standalone/eggroll/spark - federation: 通信引擎, 可填:standalone/rollsite/pulsar/rabbitmq/osx - storage: 存储引擎, 可填:standalone/eggroll -federation: 通信服务详细地址 -``` -##### 1.3.2 配置修改 -- 根据实际部署情况修改系统配置service_conf.yaml -- 修改fate_flow/bin/init_env.sh, 参考如下： -```yaml -export EGGROLL_HOME=/data/projects/fate/eggroll -export PYTHONPATH=/data/projects/fate2.0/python:/data/projects/fate2.0/fate_flow/python:/data/projects/fate/eggroll/python -venv=/data/projects/fate2.0/common/python/venv -export PATH=$PATH:$JAVA_HOME/bin -source ${venv}/bin/activate -``` - -#### 1.4 服务启停 -- init环境变量 - ```shell - source /data/projects/fate2.0/fate_flow/bin/init_env.sh - ``` -- 启动服务 - ```shell - sh /data/projects/fate2.0/fate_flow/bin/service.sh start - ``` -- 重启服务 - ```shell - sh /data/projects/fate2.0/fate_flow/bin/service.sh restart - ``` -- 停止服务 - ```shell - sh /data/projects/fate2.0/fate_flow/bin/service.sh stop - ``` -- 查询服务状态 - ```shell - sh /data/projects/fate2.0/fate_flow/bin/service.sh status - ``` - -### 2. 使用指南 -#### 2.1 数据上传 -- 若计算引擎使用standalone，reader组件参数支持配置文件路径，数据无需上传。提交任务时reader参数如下： -```yaml -reader_0: - inputs: - parameters: - delimiter: ',' - dtype: float32 - format: csv - id_name: id - label_name: y - label_type: float32 - path: file:///data/projects/fate/fateflow/examples/data/breast_hetero_guest.csv -``` -- 若计算引擎使用eggroll，需要先将数据上传至eggroll中，可参考：[eggroll数据上传](../examples/test/data.py)、[上传参数](../examples/upload/upload_guest.json)。 提交任务时reader参数： -```yaml -reader_0: - inputs: - parameters: - path: eggroll:///experiment/guest - format: raw_table -``` -#### 2.2 任务操作 -##### 2.2.1 提交任务 -- 任务配置参考[dag配置](../examples/lr/standalone/lr_train_dag.yaml) -```python -import requests -from ruamel import yaml - -base = "http://127.0.0.1:9380/v2" - -def submit_job(): - uri = "/job/submit" - dag = yaml.safe_load(open("lr_train_dag.yaml", "r")) - response = requests.post(base+uri, json={"dag_schema": dag}) - print(response.text) - ``` -##### 2.2.2 查询job -```python -import requests - -base = "http://127.0.0.1:9380/v2" - -def query_job(job_id): - uri = "/job/query" - response = requests.post(base+uri, json={"job_id": job_id}) - print(response.text) -``` -##### 2.2.3 查询task -```python -import requests - -base = "http://127.0.0.1:9380/v2" - -def query_task(job_id, role, party_id, task_name): - uri = "/job/task/query" - response = requests.post(base+uri, json={"job_id": job_id, "role": role, "party_id": party_id, "task_name": task_name}) - print(response.text) -``` - -##### 2.2.4 停止任务 -```python -import requests - -base = "http://127.0.0.1:9380/v2" - -def stop_job(job_id): - uri = "/job/stop" - response = requests.post(base+uri, json={"job_id": job_id}) - print(response.text) -``` - -#### 2.3 输出查询 -##### 2.3.1 metric -```python -import requests - -base = "http://127.0.0.1:9380/v2" - -def metric_query(job_id, role, party_id, task_name): - uri = "/output/metric/query" - data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "task_name": task_name - } - response = requests.get(base+uri, params=data) - print(response.text) -``` -##### 2.3.2 model -```python -import requests - -base = "http://127.0.0.1:9380/v2" - -def model_query(job_id, role, party_id, task_name): - uri = "/output/model/query" - data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "task_name": task_name - } - response = requests.get(base+uri, params=data) - print(response.text) -``` - -#### 2.4 算法容器 -##### 2.4.1 方案 -算法容器化方案参考：[算法容器注册与加载方案](./container.md) - -##### 2.4.2 配置 -service_conf.yaml中默认配置如下： -```yaml -worker: - type: native - docker: - config: - base_url: unix:///var/run/docker.sock - image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha - # 容器内路径，一般不需要更改 - fate_root_dir: /data/projects/fate - # 宿主机路径，根据实际情况填写 - eggroll_conf_dir: - k8s: - image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha - namespace: fate-10000 -``` -- 在 2.0.0-alpha 版本中暂不支持算法容器注册功能，只支持固定模式的算法运行方案：`local`、`docker` 或 `k8s`, 由配置 `type` 决定运行模式。 -- `worker.type` 支持：`docker`、`k8s`，默认使用非容器模式，即 `native`。 -- 容器模式不支持通信组件使用 `standalone`，需更改 `default_engines.federation` 为其他组件。 +# Quick Start + +## 1. Environment Setup +You can choose one of the following three deployment modes based on your requirements: + +### 1.1 Pypi Package Installation +Note: This mode operates in a single-machine mode. + +#### 1.1.1 Installation +- Prepare and install [conda](https://docs.conda.io/projects/miniconda/en/latest/) environment. +- Create a virtual environment: +```shell +# FATE requires Python >= 3.8 +conda create -n fate_env python=3.8 +conda activate fate_env +``` +- Install FATE Flow and related dependencies: +```shell +pip install fate_client[fate,fate_flow]==2.0.0.b0 +``` + +#### 1.1.2 Service Initialization +```shell +fate_flow init --ip 127.0.0.1 --port 9380 --home $HOME_DIR +``` +- `ip`: The IP address where the service runs. +- `port`: The HTTP port the service runs on. +- `home`: The data storage directory, including data, models, logs, job configurations, and SQLite databases. + +#### 1.1.3 Service Start/Stop +```shell +fate_flow status/start/stop/restart +``` + +### 1.2 Standalone Deployment +Refer to [Standalone Deployment](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/standalone-deploy/README.zh.md). + +### 1.3 Cluster Deployment +Refer to [Allinone Deployment](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/cluster-deploy/allinone/fate-allinone_deployment_guide.zh.md). + +## 2. User Guide +FATE provides client tools including SDK, CLI, and Pipeline. If you don't have FATE Client deployed in your environment, you can download it using `pip install fate_client`. The following operations are based on CLI. + +### 2.1 Data Upload +In version 2.0-beta, data uploading is a two-step process: + +- **upload**: Uploads data to FATE-supported storage services. +- **transformer**: Transforms data into a DataFrame. + +#### 2.1.1 upload +##### 2.1.1.1 Configuration and Data +- Upload configuration can be found at [examples-upload](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/upload), and the data is located at [upload-data](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/data). +- You can also use your own data and modify the "meta" information in the upload configuration. + +##### 2.1.1.2 Upload Guest Data +```shell +flow data upload -c examples/upload/upload_guest.json +``` +- Record the returned "name" and "namespace" for use in the transformer phase. + +##### 2.1.1.3 Upload Host Data +```shell +flow data upload -c examples/upload/upload_host.json +``` +- Record the returned "name" and "namespace" for use in the transformer phase. + +##### 2.1.1.4 Upload Result +```json +{ + "code": 0, + "data": { + "name": "36491bc8-3fef-11ee-be05-16b977118319", + "namespace": "upload" + }, + "job_id": "202308211451535620150", + "message": "success" +} +``` +Where "namespace" and "name" identify the data in FATE for future reference in the transformer phase. + +##### 2.1.1.5 Data Query +Since upload is an asynchronous operation, you need to confirm if it was successful before proceeding to the next step. +```shell +flow table query --namespace upload --name 36491bc8-3fef-11ee-be05-16b977118319 +``` +If the returned code is 0, the upload was successful. + +#### 2.1.2 Transformer +##### 2.1.2.1 Configuration +- Transformer configuration can be found at [examples-transformer](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/transformer). + +##### 2.1.2.2 Transform Guest Data +- Configuration path: examples/transformer/transformer_guest.json +- Modify the "namespace" and "name" in the "data_warehouse" section to match the output from the guest data upload. +```shell +flow data transformer -c examples/transformer/transformer_guest.json +``` + +##### 2.1.2.3 Transform Host Data +- Configuration path: examples/transformer/transformer_host.json +- Modify the "namespace" and "name" in the "data_warehouse" section to match the output from the host data upload. +```shell +flow data transformer -c examples/transformer/transformer_host.json +``` + +##### 2.1.2.4 Transformer Result +```json +{ + "code": 0, + "data": { + "name": "breast_hetero_guest", + "namespace": "experiment" + }, + "job_id": "202308211557455662860", + "message": "success" +} +``` +Where "namespace" and "name" identify the data in FATE for future modeling jobs. + +##### 2.1.2.5 Check if Data Upload Was Successful +Since the transformer is also an asynchronous operation, you need to confirm if it was successful before proceeding. +```shell +flow table query --namespace experiment --name breast_hetero_guest +``` +```shell +flow table query --namespace experiment --name breast_hetero_host +``` +If the returned code is 0, the upload was successful. + +### 2.2 Starting FATE Jobs +#### 2.2.1 Submitting a Job +Once your data is prepared, you can start submitting jobs to FATE Flow: + +- The configuration for training jobs can be found in [lr-train](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/train_lr.yaml). +- The configuration for prediction jobs can be found in [lr-predict](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/predict_lr.yaml). To use it, modify the "dag.conf.model_warehouse" to point to the output model of your training job. +- In the training and prediction job configurations, the site IDs are set to "9998" and "9999." If your deployment environment is the cluster version, you need to replace them with the actual site IDs. For the standalone version, you can use the default configuration. +- If you want to use your own data, you can change the "namespace" and "name" of "data_warehouse" for both the guest and host in the configuration. +- To submit a job, use the following command: +```shell +flow job submit -c examples/lr/train_lr.yaml +``` +- A successful submission will return the following result: +```json +{ + "code": 0, + "data": { + "model_id": "202308211911505128750", + "model_version": "0" + }, + "job_id": "202308211911505128750", + "message": "success" +} +``` +The "data" section here contains the output model of the job. + +#### 2.2.2 Querying a Job +While a job is running, you can check its status using the query command: +```shell +flow job query -j $job_id +``` + +#### 2.2.3 Stopping a Job +During job execution, you can stop the current job using the stop command: +```shell +flow job stop -j $job_id +``` + +#### 2.2.4 Rerunning a Job +If a job fails during execution, you can rerun it using the rerun command: +```shell +flow job rerun -j $job_id +``` + +### 2.3 Obtaining Job Outputs +Job outputs include data, models, and metrics. + +#### 2.3.1 Output Metrics +To query output metrics, use the following command: +```shell +flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name +``` +For example, if you used the training DAG from above, you can use `flow output query-metric -j 202308211911505128750 -r arbiter -p 9998 -tn lr_0` to query metrics. +The query result will look like this: +```json +{ + "code": 0, + "data": [ + { + "data": [ + { + "metric": [ + 0.0 + ], + "step": 0, + "timestamp": 1692616428.253495 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + }, + { + "data": [ + { + "metric": [ + -0.07785049080848694 + ], + "step": 1, + "timestamp": 1692616432.9727712 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + } + ], + "message": "success" +} +``` + +#### 2.3.2 Output Models +##### 2.3.2.1 Querying Models +To query output models, use the following command: +```shell +flow output query-model -j $job_id -r $role -p $party_id -tn $task_name +``` +For example, if you used the training DAG from above, you can use `flow output query-model -j 202308211911505128750 -r host -p 9998 -tn lr_0` to query models. +The query result will be similar to this: + +```json +{ + "code": 0, + "data": [ + { + "model": { + "file": "202308211911505128750_host_9998_lr_0", + "namespace": "202308211911505128750_host_9998_lr_0" + }, + "name": "HeteroLRHost_9998_0", + "namespace": "202308211911505128750_host_9998_lr_0", + "role": "host", + "party_id": "9998", + "work_mode": 1 + } + ], + "message": "success" +} +``` + +##### 2.3.2.2 Downloading Models +To download models, use the following command: +```shell +flow output download-model -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +For example, if you used the training DAG from above, you can use `flow output download-model -j 202308211911505128750 -r host -p 9998 -tn lr_0 -o ./` to download the model. +The download result will be similar to this: + +```json +{ + "code": 0, + "directory": "./output_model_202308211911505128750_host_9998_lr_0", + "message": "download success, please check the path: ./output_model_202308211911505128750_host_9998_lr_0" +} +``` + +#### 2.3.3 Output Data +##### 2.3.3.1 Querying Data Tables +To query output data tables, use the following command: +```shell +flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name +``` +For example, if you used the training DAG from above, you can use `flow output query-data-table -j 202308211911505128750 -r host -p 9998 -tn binning_0` to query data tables. +The query result will be similar to this: + +```json +{ + "train_output_data": [ + { + "name": "9e28049c401311ee85c716b977118319", + "namespace": "202308211911505128750_binning_0" + } + ] +} +``` + +##### 2.3.3.2 Preview Data +```shell +flow output display-data -j $job_id -r $role -p $party_id -tn $task_name +``` +To preview output data using the above training DAG submission, you can use the following command: `flow output display-data -j 202308211911505128750 -r host -p 9998 -tn binning_0`. + +##### 2.3.3.3 Download Data +```shell +flow output download-data -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +To download output data using the above training DAG submission, you can use the following command: `flow output download-data -j 202308211911505128750 -r guest -p 9999 -tn lr_0 -o ./`. + +The download result will be as follows: +```json +{ + "code": 0, + "directory": "./output_data_202308211911505128750_guest_9999_lr_0", + "message": "download success, please check the path: ./output_data_202308211911505128750_guest_9999_lr_0" +} +``` + +## 3. More Documentation +- [Restful-api](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/doc/swagger/swagger.yaml) +- [CLI](https://github.com/FederatedAI/FATE-Client/tree/v2.0.0-beta/python/fate_client/flow_cli/build/doc) +- [Pipeline](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/tutorial) +- [FATE Quick Start](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/quick_start.md) +- [FATE Algorithms](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/fate) \ No newline at end of file diff --git a/doc/quick_start.zh.md b/doc/quick_start.zh.md new file mode 100644 index 000000000..90176526b --- /dev/null +++ b/doc/quick_start.zh.md @@ -0,0 +1,562 @@ +# 快速入门 + +## 1. 环境部署 +以下三种模式可根据需求自行选择一种 +### 1.1 Pypi包安装 +说明：此方式的运行模式为单机模式 +#### 1.1.1 安装 +- [conda](https://docs.conda.io/projects/miniconda/en/latest/)环境准备及安装 +- 创建虚拟环境 +```shell +# fate的运行环境为python>=3.8 +conda create -n fate_env python=3.8 +conda activate fate_env +``` +- 安装fate flow及相关依赖 +```shell +pip install fate_client[fate,fate_flow]==2.0.0.b0 +``` + +#### 1.1.2 服务初始化 +```shell +fate_flow init --ip 127.0.0.1 --port 9380 --home $HOME_DIR +``` +- ip: 服务运行ip +- port：服务运行时的http端口 +- home: 数据存储目录。主要包括：数据/模型/日志/作业配置/sqlite.db等内容 + +#### 1.1.3 服务启停 +```shell +fate_flow status/start/stop/restart +``` + +### 1.2 单机版部署 +参考[单机版部署](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/standalone-deploy/README.zh.md) + +### 1.3 集群部署 +参考[allinone部署](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/cluster-deploy/allinone/fate-allinone_deployment_guide.zh.md) + +## 2. 使用指南 +fate提供的客户端包括SDK、CLI和Pipeline，若你的环境中没有部署FATE Client,可以使用`pip install fate_client`下载，以下的使用操作均基于cli编写。 + +### 2.1 数据上传 +在2.0-beta版本中，数据上传分为两步： +- upload: 将数据上传到FATE支持存储服务中 +- transformer: 将数据转化成dataframe +#### 2.1.1 upload +#### 2.1.1.1 配置及数据 + - 上传配置位于[examples-upload](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/upload)，上传数据位于[upload-data](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/data) + - 你也可以使用自己的数据，并修改upload配置中的"meta"信息。 +#### 2.1.1.2 上传guest方数据 +```shell +flow data upload -c examples/upload/upload_guest.json +``` +- 需要记录返回的name和namespace，作为transformer的参数。 +#### 2.1.1.3 上传host方数据 +```shell +flow data upload -c examples/upload/upload_host.json +``` +- 需要记录返回的name和namespace，作为transformer的参数。 +#### 2.1.1.4 上传结果 +```json +{ + "code": 0, + "data": { + "name": "36491bc8-3fef-11ee-be05-16b977118319", + "namespace": "upload" + }, + "job_id": "202308211451535620150", + "message": "success" +} +``` +其中"namespace"和"name"是这份数据在fate中的标识，以便下面后续transformer阶段使用时可直接引用。 + +#### 2.1.1.5 数据查询 +因为upload为异步操作，需要确认是否上传成功才可进行后续操作。 +```shell +flow table query --namespace upload --name 36491bc8-3fef-11ee-be05-16b977118319 +``` +上传成功信息如下： +```json +{ + "code": 0, + "data": { + "count": 569, + "data_type": "table", + "engine": "standalone", + "meta": { + "delimiter": ",", + "dtype": "'float32", + "header": "extend_sid,id,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19", + "input_format": "dense", + "label_type": "int", + "match_id_name": "id", + "match_id_range": 0, + "sample_id_name": "extend_sid", + "tag_value_delimiter": ":", + "tag_with_value": false, + "weight_type": "float32" + }, + "name": "36491bc8-3fef-11ee-be05-16b977118319", + "namespace": "upload", + "path": "xxx", + "source": { + "component": "upload", + "output_artifact_key": "data", + "output_index": null, + "party_task_id": "", + "task_id": "", + "task_name": "upload" + } + }, + "message": "success" +} + +``` +若返回的code为0即为上传成功。 + +#### 2.1.2 transformer +#### 2.1.2.1 配置 + - transformer配置位于[examples-transformer](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/transformer) +#### 2.1.2.2 transformer guest +- 配置路径位于： examples/transformer/transformer_guest.json +- 修改配置中"data_warehouse"的"namespace"和"name"：上面upload guest阶段的输出 +```shell +flow data transformer -c examples/transformer/transformer_guest.json +``` +#### 2.1.2.3 transformer host +- 配置路径位于： examples/transformer/transformer_host.json +- 修改配置中"data_warehouse"的"namespace"和"name"：上面upload host阶段的输出 +```shell +flow data transformer -c examples/transformer/transformer_host.json +``` +#### 2.1.2.4 transformer结果 +```json +{ + "code": 0, + "data": { + "name": "breast_hetero_guest", + "namespace": "experiment" + }, + "job_id": "202308211557455662860", + "message": "success" +} +``` +其中"namespace"和"name"是这份数据在fate中的标识，后续建模作业中使用。 + +#### 2.1.2.5 查看数据是否上传成功 + +因为transformer也是异步操作，需要确认是否上传成功才可进行后续操作。 +```shell +flow table query --namespace experiment --name breast_hetero_guest +``` +```shell +flow table query --namespace experiment --name breast_hetero_host +``` +若返回的code为0即为上传成功。 + +### 2.2 开始FATE作业 +#### 2.2.1 提交作业 +当你的数据准备好后，可以开始提交作业给FATE Flow： +- 训练job配置example位于[lr-train](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/train_lr.yaml); +- 预测job配置example位于[lr-predict](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/predict_lr.yaml);预测任务需要修改"dag.conf.model_warehouse"成训练作业的输出模型。 +- 训练和预测job配置中站点id为"9998"和"9999"。如果你的部署环境为集群版，需要替换成真实的站点id；单机版可使用默认配置。 +- 如果想要使用自己的数据，可以更改配置中guest和host的data_warehouse的namespace和name +- 提交作业的命令为: +```shell +flow job submit -c examples/lr/train_lr.yaml +``` +- 提交成功返回结果: +```json +{ + "code": 0, + "data": { + "model_id": "202308211911505128750", + "model_version": "0" + }, + "job_id": "202308211911505128750", + "message": "success" +} + +``` +这里的"data"内容即为该作业的输出模型。 + +#### 2.2.2 查询作业 +在作业的运行过程时，你可以通过查询命令获取作业的运行状态 +```shell +flow job query -j $job_id +``` + +#### 2.2.3 停止作业 +在作业的运行过程时，你可以通过停止作业命令来终止当前作业 +```shell +flow job stop -j $job_id +``` + +#### 2.2.4 重跑作业 +在作业的运行过程时，如果运行失败，你可以通过重跑命令来重跑当前作业 +```shell +flow job rerun -j $job_id +``` + +### 2.3 获取作业输出结果 +作业的输出包括数据、模型和指标 +#### 2.3.1 输出指标 +查询输出指标命令： +```shell +flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output query-metric -j 202308211911505128750 -r arbiter -p 9998 -tn lr_0`查询。 +查询结果如下: +```json +{ + "code": 0, + "data": [ + { + "data": [ + { + "metric": [ + 0.0 + ], + "step": 0, + "timestamp": 1692616428.253495 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + }, + { + "data": [ + { + "metric": [ + -0.07785049080848694 + ], + "step": 1, + "timestamp": 1692616432.9727712 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + } + ], + "message": "success" +} + +``` + + +#### 2.3.2 输出模型 +##### 2.3.2.1 查询模型 +```shell +flow output query-model -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output query-model -j 202308211911505128750 -r host -p 9998 -tn lr_0`查询。 +查询结果如下： +```json +{ + "code": 0, + "data": { + "output_model": { + "data": { + "estimator": { + "end_epoch": 10, + "is_converged": false, + "lr_scheduler": { + "lr_params": { + "start_factor": 0.7, + "total_iters": 100 + }, + "lr_scheduler": { + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.07269999999999996 + ], + "_step_count": 10, + "base_lrs": [ + 0.1 + ], + "end_factor": 1.0, + "last_epoch": 9, + "start_factor": 0.7, + "total_iters": 100, + "verbose": false + }, + "method": "linear" + }, + "optimizer": { + "alpha": 0.001, + "l1_penalty": false, + "l2_penalty": true, + "method": "sgd", + "model_parameter": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "model_parameter_dtype": "float32", + "optim_param": { + "lr": 0.1 + }, + "optimizer": { + "param_groups": [ + { + "dampening": 0, + "differentiable": false, + "foreach": null, + "initial_lr": 0.1, + "lr": 0.07269999999999996, + "maximize": false, + "momentum": 0, + "nesterov": false, + "params": [ + 0 + ], + "weight_decay": 0 + } + ], + "state": {} + } + }, + "param": { + "coef_": [ + [ + -0.10828543454408646 + ], + [ + -0.07341302931308746 + ], + [ + -0.10850320011377335 + ], + [ + -0.10066638141870499 + ], + [ + -0.04595951363444328 + ], + [ + -0.07001449167728424 + ], + [ + -0.08949052542448044 + ], + [ + -0.10958756506443024 + ], + [ + -0.04012322425842285 + ], + [ + 0.02270071767270565 + ], + [ + -0.07198350876569748 + ], + [ + 0.00548586156219244 + ], + [ + -0.06599288433790207 + ], + [ + -0.06410090625286102 + ], + [ + 0.016374297440052032 + ], + [ + -0.01607361063361168 + ], + [ + -0.011447405442595482 + ], + [ + -0.04352564364671707 + ], + [ + 0.013161249458789825 + ], + [ + 0.013506329618394375 + ] + ], + "dtype": "float32", + "intercept_": null + } + } + }, + "meta": { + "batch_size": null, + "epochs": 10, + "init_param": { + "fill_val": 0.0, + "fit_intercept": false, + "method": "zeros", + "random_state": null + }, + "label_count": false, + "learning_rate_param": { + "method": "linear", + "scheduler_params": { + "start_factor": 0.7, + "total_iters": 100 + } + }, + "optimizer_param": { + "alpha": 0.001, + "method": "sgd", + "optimizer_params": { + "lr": 0.1 + }, + "penalty": "l2" + }, + "ovr": false + } + } + }, + "message": "success" +} + +``` + +##### 2.3.2.2 下载模型 +```shell +flow output download-model -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +如使用上面的训练dag提交任务，可以使用`flow output download-model -j 202308211911505128750 -r host -p 9998 -tn lr_0 -o ./`下载。 +下载结果如下： +```json +{ + "code": 0, + "directory": "./output_model_202308211911505128750_host_9998_lr_0", + "message": "download success, please check the path: ./output_model_202308211911505128750_host_9998_lr_0" +} + + +``` + + +#### 2.3.3 输出数据 +##### 2.3.3.1 查询数据表 +```shell +flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output query-data-table -j 202308211911505128750 -r host -p 9998 -tn binning_0`查询。 +查询结果如下： +```json +{ + "train_output_data": [ + { + "name": "9e28049c401311ee85c716b977118319", + "namespace": "202308211911505128750_binning_0" + } + ] +} +``` + +##### 2.3.3.2 预览数据 +```shell +flow output display-data -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output display-data -j 202308211911505128750 -r host -p 9998 -tn binning_0`预览输出数据。 + +##### 2.3.3.3 下载数据 +```shell +flow output download-data -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +如使用上面的训练dag提交任务，可以使用`flow output download-data -j 202308211911505128750 -r guest -p 9999 -tn lr_0 -o ./`下载输出数据。 +下载结果如下： +```json +{ + "code": 0, + "directory": "./output_data_202308211911505128750_guest_9999_lr_0", + "message": "download success, please check the path: ./output_data_202308211911505128750_guest_9999_lr_0" +} + +``` + +## 3.更多文档 +- [Restful-api](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/doc/swagger/swagger.yaml) +- [CLI](https://github.com/FederatedAI/FATE-Client/tree/v2.0.0-beta/python/fate_client/flow_cli/build/doc) +- [Pipeline](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/tutorial) +- [FATE快速开始](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/quick_start.md) +- [FATE算法](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/fate) diff --git a/doc/swagger/index.md b/doc/swagger/index.md new file mode 100644 index 000000000..5bb1f36d4 --- /dev/null +++ b/doc/swagger/index.md @@ -0,0 +1,3 @@ +## Swagger API + +!!swagger swagger.yaml!! diff --git a/doc/swagger/swagger.yaml b/doc/swagger/swagger.yaml new file mode 100644 index 000000000..03cbaba00 --- /dev/null +++ b/doc/swagger/swagger.yaml @@ -0,0 +1,1999 @@ +swagger: '2.0' +basePath: /v2 +paths: + /client/client/create: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: post_create_client_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqAppName' + tags: + - client + /client/client/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseStatus' + operationId: post_delete_client_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqAppId' + tags: + - client + /client/client/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: get_client_QueryClientApp + parameters: + - required: false + description: App ID for the client + in: query + name: app_id + type: string + - required: false + description: App name for the client + in: query + name: app_name + type: string + tags: + - client + /client/partner/create: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponsePartnerInfo' + operationId: post_create_partner_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqPartnerCreate' + tags: + - client + /client/partner/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseStatus' + operationId: post_delete_partner_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqPartyId' + tags: + - client + /client/partner/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: get_client_QueryPartnerApp + parameters: + - required: false + description: Site ID + in: query + name: party_id + type: string + tags: + - client + /client/site/create: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: post_create_site_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqPartyId' + tags: + - client + /client/site/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseStatus' + operationId: post_delete_site_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqPartyId' + tags: + - client + /client/site/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: get_client_QuerySiteApp + parameters: + - required: true + description: Site ID + in: query + name: party_id + type: string + tags: + - client + /data/component/dataframe/transformer: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseUploadData' + operationId: post_transformer_data + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqTransformerData' + tags: + - data + /data/component/download: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseComponentDownload' + operationId: post_download_data + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqComponentDownload' + tags: + - data + /data/component/upload: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseUploadData' + operationId: post_upload_data + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqComponentUpload' + tags: + - data + /data/download: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseUploadData' + operationId: get_data_Download + parameters: + - required: true + description: Name of the data table + in: query + name: name + type: string + - required: true + description: Namespace of the data table + in: query + name: namespace + type: string + - required: false + description: Whether the first row of the file is the data's head + in: query + name: header + type: string + tags: + - data + /job/clean: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseJobClean' + operationId: post_clean_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' + tags: + - job + /job/dag/dependency: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_job_DagDependency + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + tags: + - job + /job/list/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_job_QueryJobList + parameters: + - required: false + description: Limit of rows or entries + in: query + name: limit + type: string + - required: false + description: Page number + in: query + name: page + type: string + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: Description information + in: query + name: description + type: string + - required: false + description: Participant information + in: query + name: partner + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Status of the job or task + in: query + name: status + type: string + - required: false + description: Field name for sorting + in: query + name: order_by + type: string + - required: false + description: 'Sorting order: asc/desc' + in: query + name: order + type: string + - required: false + description: Username provided by the upper-level system + in: header + name: user_name + type: string + tags: + - job + /job/log/download: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_download_job_logs + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' + produces: + - application/gzip + tags: + - job + /job/notes/add: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_add_notes + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqNodesAdd' + tags: + - job + /job/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonListToDictModel' + operationId: get_job_QueryJob + parameters: + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Status of the job or task + in: query + name: status + type: string + - required: false + description: Username provided by the upper-level system + in: header + name: user_name + type: string + tags: + - job + /job/queue/clean: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseJobClean' + operationId: post_clean_queue + tags: + - job + /job/rerun: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_request_rerun_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' + tags: + - job + /job/stop: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_request_stop_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' + tags: + - job + /job/submit: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseJobSubmit' + operationId: post_submit_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqSubmitJob' + tags: + - job + /job/task/list/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_job_QueryTaskList + parameters: + - required: false + description: Limit of rows or entries + in: query + name: limit + type: string + - required: false + description: Page number + in: query + name: page + type: string + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Field name for sorting + in: query + name: order_by + type: string + - required: false + description: 'Sorting order: asc/desc' + in: query + name: order + type: string + tags: + - job + /job/task/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonListToDictModel' + operationId: get_job_QueryTask + parameters: + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Status of the job or task + in: query + name: status + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Task ID + in: query + name: task_id + type: string + - required: false + description: Task version + in: query + name: task_version + type: string + tags: + - job + /log/count: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonDataModel' + operationId: get_log_Count + parameters: + - required: true + description: Log level or type + in: query + name: log_type + type: string + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Instance ID of the FATE Flow service + in: query + name: instance_id + type: string + tags: + - log + /log/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonListDictModel' + operationId: get_log_Get + parameters: + - required: true + description: Log level or type + in: query + name: log_type + type: string + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Starting line number + in: query + name: begin + type: string + - required: false + description: Ending line number + in: query + name: end + type: string + - required: false + description: Instance ID of the FATE Flow service + in: query + name: instance_id + type: string + tags: + - log + /model/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ModelDelete' + operationId: post_delete_model + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqModelDelete' + tags: + - model + /model/export: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_export + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqModelExport' + tags: + - model + /model/import: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_import_model + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqModelImport' + tags: + - model + /model/load: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_load + tags: + - model + /model/migrate: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_migrate + tags: + - model + /model/restore: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_restore + tags: + - model + /model/store: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_store + tags: + - model + /output/data/display: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseTableDisplay' + operationId: get_output_OutputDataDisplay + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + tags: + - output + /output/data/download: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: get_output_OutputDataDownload + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + - required: false + description: Primary key for output data or model of the task + in: query + name: output_key + type: string + produces: + - application/gzip + tags: + - output + /output/data/table: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseDataTable' + operationId: get_output_OutputDataTable + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + tags: + - output + /output/metric/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseDeleteMetric' + operationId: post_delete_metric + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqMetricDelete' + tags: + - output + /output/metric/key/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/OutputQuery' + operationId: get_output_QueryMetricKey + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + tags: + - output + /output/metric/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_output_QueryMetric + parameters: + - required: true + description: Site ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + - required: false + description: Filter conditions + in: query + name: filters + type: string + tags: + - output + /output/model/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_output_delete_model + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqMetricDelete' + tags: + - output + /output/model/download: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: get_output_Download + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + produces: + - application/x-tar + tags: + - output + /output/model/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_output_QueryModel + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + tags: + - output + /permission/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseGrantPermission' + operationId: post_delete + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqGrant' + tags: + - permission + /permission/grant: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseGrantPermission' + operationId: post_grant + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqGrant' + tags: + - permission + /permission/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponsePermissionModel' + operationId: get_permission_Query + parameters: + - required: true + description: App ID + in: query + name: app_id + type: string + tags: + - permission + /permission/resource/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_delete_resource_permission + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqResourceGrant' + tags: + - permission + /permission/resource/grant: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_grant_resource_permission + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqResourceGrant' + tags: + - permission + /permission/resource/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseResourceModel' + operationId: get_permission_QueryResourcePrivilege + parameters: + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Component name + in: query + name: component + type: string + - required: false + description: List of datasets + in: query + name: dataset + type: string + tags: + - permission + /permission/role/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonListModel' + operationId: get_permission_QueryRoles + tags: + - permission + /provider/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseGrantPermission' + operationId: post_provider_Delete + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqProviderDelete' + tags: + - provider + /provider/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_provider_Query + parameters: + - required: false + description: Component provider name + in: query + name: name + type: string + - required: false + description: Component running mode + in: query + name: device + type: string + - required: false + description: Component version + in: query + name: version + type: string + - required: false + description: >- + Registered algorithm full name, provider + ':' + version + '@' + + running mode, e.g., fate:2.0.0@local + in: query + name: provider_name + type: string + tags: + - provider + /provider/register: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_register + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqProviderRegister' + tags: + - provider + /server/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_delete_server + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServerDelete' + tags: + - server + /server/fateflow: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_server_FateFlowServerInfo + tags: + - server + /server/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonListModel' + operationId: get_server_QueryServer + parameters: + - required: true + description: Server name + in: query + name: server_name + type: string + tags: + - server + /server/query/all: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_server_QueryAll + tags: + - server + /server/registry: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseServerRegister' + operationId: post_register_server + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServerRegistry' + tags: + - server + /server/service/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_delete_service + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServiceRegistry' + tags: + - server + /server/service/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_server_QueryService + parameters: + - required: true + description: Server name + in: query + name: server_name + type: string + - required: true + description: Service name + in: query + name: service_name + type: string + tags: + - server + /server/service/registry: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_registry_service + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServiceRegistry' + tags: + - server + /site/info/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_site_QuerySiteInfo + tags: + - site + /table/bind/path: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_bind_path + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqBindPath' + tags: + - table + /table/delete: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_delete_table + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServiceRegistry' + tags: + - table + /table/query: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_table_QueryTable + parameters: + - required: true + description: Namespace of the data table + in: query + name: namespace + type: string + - required: true + description: Name of the data table + in: query + name: name + type: string + - required: false + description: Whether to return preview data + in: query + name: display + type: string + tags: + - table +info: + title: FATE Flow restful api + version: 2.0.0-beta +produces: + - application/json +consumes: + - application/json +tags: + - name: client + description: client-Related Operations + - name: data + description: data-Related Operations + - name: job + description: job-Related Operations + - name: log + description: log-Related Operations + - name: model + description: model-Related Operations + - name: output + description: output-Related Operations + - name: permission + description: permission-Related Operations + - name: provider + description: provider-Related Operations + - name: server + description: server-Related Operations + - name: site + description: site-Related Operations + - name: table + description: table-Related Operations +definitions: + ReqAppName: + required: + - app_name + properties: + app_name: + type: string + description: App name for the client + type: object + ResponseClientInfo: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/ClientInfo' + type: object + CommonModel: + properties: + message: + type: string + code: + type: integer + type: object + ClientInfo: + properties: + app_name: + type: string + app_id: + type: string + app_token: + type: string + app_type: + type: string + type: object + ReqAppId: + required: + - app_id + properties: + app_id: + type: string + description: App ID for the client + type: object + ResponseStatus: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/Status' + type: object + Status: + properties: + status: + type: integer + type: object + ReqPartyId: + required: + - party_id + properties: + party_id: + type: string + description: Site ID + type: object + ReqPartnerCreate: + required: + - app_id + - app_token + - party_id + properties: + party_id: + type: string + description: Site ID + app_id: + type: string + description: App ID for the site + app_token: + type: string + description: App token for the site + type: object + ResponsePartnerInfo: + properties: + message: + type: string + code: + type: integer + x-mask: '{data}' + type: object + ReqComponentUpload: + required: + - file + - head + - partitions + properties: + file: + type: string + description: File path on the server + head: + type: boolean + description: Whether the first row of the file is the data's head + partitions: + type: integer + description: Number of data partitions + meta: + type: object + description: Metadata of the data + extend_sid: + type: boolean + description: Whether to automatically fill a column as data row ID + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + type: object + ResponseUploadData: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/dataInfo' + job_id: + type: string + type: object + dataInfo: + properties: + name: + type: string + namespace: + type: string + type: object + ReqComponentDownload: + required: + - name + - namespace + properties: + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + path: + type: string + description: File path on the server + type: object + ResponseComponentDownload: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/ComponentInfo' + job_id: + type: string + type: object + ComponentInfo: + properties: + name: + type: string + namespace: + type: string + path: + type: string + type: object + ReqTransformerData: + required: + - name + - namespace + properties: + data_warehouse: + type: object + description: 'Data output, content like: {name: xxx, namespace: xxx}' + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + drop: + type: boolean + description: Whether to destroy data if it already exists + type: object + ReqSubmitJob: + required: + - dag_schema + properties: + dag_schema: + type: object + description: >- + Definition and configuration of jobs, including the configuration of + multiple tasks + user_name: + type: string + description: Name of the data table + type: object + ResponseJobSubmit: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/JobInfo' + job_id: + type: string + type: object + JobInfo: + properties: + model_id: + type: string + model_version: + type: string + type: object + CommonListToDictModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: array + items: + type: object + type: object + ReqJobId: + required: + - job_id + properties: + job_id: + type: string + description: Job ID + type: object + CommonDictModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: object + type: object + ResponseJobClean: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/queueInfo' + type: object + queueInfo: + properties: + job_id: + type: string + type: object + ReqNodesAdd: + required: + - job_id + - notes + - party_id + - role + properties: + job_id: + type: string + description: Job ID + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + party_id: + type: string + description: Site ID + notes: + type: string + description: Tags and customizable information for tasks + type: object + CommonDataModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: integer + type: object + CommonListDictModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: array + items: + type: object + type: object + ReqModelExport: + required: + - model_id + - model_version + - party_id + - path + - role + properties: + model_id: + type: string + description: Model ID + model_version: + type: string + description: Model version + party_id: + type: string + description: Site ID + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + path: + type: string + description: Directory path on the server + type: object + ReqModelImport: + required: + - model_id + - model_version + properties: + model_id: + type: string + description: Model ID + model_version: + type: string + description: Model version + type: object + ReqModelDelete: + allOf: + - $ref: '#/definitions/ReqModelImport' + - properties: + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + party_id: + type: string + description: Site ID + task_name: + type: string + description: Task name + output_key: + type: string + description: Primary key for output data or model of the task + type: object + ModelDelete: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/Delete' + type: object + Delete: + properties: + count: + type: boolean + type: object + OutputQuery: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/MetricKeyInfo' + type: object + MetricKeyInfo: + properties: + name: + type: string + step_axis: + type: string + type: + type: string + groups: + type: array + items: + type: object + type: object + ReqMetricDelete: + required: + - job_id + - role + - task_name + properties: + job_id: + type: string + description: Job ID + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + task_name: + type: string + description: Task name + type: object + ResponseDeleteMetric: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: boolean + type: object + ResponseFateFlow: + properties: + code: + type: integer + message: + type: string + data: + type: object + type: object + ResponseDataTable: + properties: + train_out_data: + type: array + items: + $ref: '#/definitions/dataInfo' + type: object + ResponseTableDisplay: + properties: + train_out_data: + type: array + items: + type: array + items: + type: string + type: object + ReqGrant: + required: + - app_id + - role + properties: + app_id: + type: string + description: App ID + role: + type: string + description: Permission name + type: object + ResponseGrantPermission: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/GrantInfo' + type: object + GrantInfo: + properties: + status: + type: boolean + type: object + ResponsePermissionModel: + properties: + data: + $ref: '#/definitions/PermissionInfo' + type: object + PermissionInfo: + properties: + client: + type: array + items: + type: object + type: object + CommonListModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: array + items: + type: string + type: object + ReqResourceGrant: + required: + - party_id + properties: + party_id: + type: string + description: Site ID + component: + type: string + description: Component name + dataset: + type: array + items: + type: object + type: object + ResponseResourceModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: object + type: object + ReqProviderRegister: + required: + - device + - metadata + - name + - version + properties: + name: + type: string + description: Component provider name + device: + type: string + description: Component running mode + version: + type: string + description: Component version + metadata: + type: string + description: Detailed information about component registration + type: object + ReqProviderDelete: + properties: + name: + type: string + description: Component provider name + device: + type: string + description: Component running mode + version: + type: string + description: Component version + provider_name: + type: string + description: >- + Registered algorithm full name, provider + ':' + version + '@' + + running mode, e.g., fate:2.0.0@local + type: object + ReqServerRegistry: + required: + - host + - port + - server_name + properties: + server_name: + type: string + description: Server name + host: + type: string + description: Host IP + port: + type: string + description: Service port + protocol: + type: string + description: 'Protocol: http/https' + type: object + ResponseServerRegister: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/RegisterInfo' + type: object + RegisterInfo: + properties: + host: + type: string + port: + type: string + protocol: + type: string + server_name: + type: string + type: object + ReqServerDelete: + required: + - server_name + properties: + server_name: + type: string + description: Server name + type: object + ReqServiceRegistry: + required: + - name + - namespace + properties: + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + type: object + ReqBindPath: + required: + - name + - namespace + - path + properties: + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + path: + type: string + description: File path on the server + type: object +responses: + ParseError: + description: When a mask can't be parsed + MaskError: + description: When any error occurs on mask \ No newline at end of file diff --git a/doc/system_conf.md b/doc/system_conf.md new file mode 100644 index 000000000..8df23fc64 --- /dev/null +++ b/doc/system_conf.md @@ -0,0 +1,216 @@ +# System Configuration +FATE Flow uses YAML to define system configurations, and the configuration file is located at: `conf/service_conf.yaml`. The specific configuration contents and their meanings are as follows: + +| Configuration Item | Description | Values | +|----------------------|------|------------------------------| +| party_id | Local site ID | For example, "9999", "10000" | +| use_registry | Whether to use a registry center; currently, only ZooKeeper mode is supported, and it requires correct ZooKeeper configuration. Note: If using high availability mode, ensure this configuration is set to true. | true/false | +| encrypt | Encryption module | See [Encryption Module](#encryption-module) | +| fateflow | Configuration for the FATE Flow service, including ports, command channel service, and proxy | See [FateFlow Configuration](#fateflow-configuration) | +| database | Configuration information for the database service | See [Database Configuration](#database-configuration) | +| default_engines | System's engine services, including computing, storage, and communication engines | See [Engine Configuration](#engine-configuration) | +| default_provider | Component source information, including provider name, component version, and execution mode | See [Default Registered Algorithm Configuration](#default-registered-algorithm-configuration) | +| federation | Communication service pool | See [Communication Engine Pool](#communication-engine-pool) | +| computing | Computing service pool | See [Computing Engine Pool](#computing-engine-pool) | +| storage | Storage service pool | See [Storage Engine Pool](#storage-configuration) | +| hook_module | Hook configuration, currently supports client authentication, site authentication, and authorization hooks | See [Hook Module Configuration](#hook-module-configuration) | +| authentication | Authentication and authorization switches | See [Authentication Switch](#authentication-switch) | +| model_store | Model storage configuration | See [Model Storage](#model-storage) | +| zookeeper | ZooKeeper service configuration | See [ZooKeeper Configuration](#zookeeper-configuration) | + +## Encryption Module +```yaml +key_0: + module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt + private_path: private_key.pem +``` +This encryption module is primarily used for encrypting passwords (e.g., MySQL passwords): +- "key_0" is the key for the encryption module (you can customize the name), making it easier to reference in other configurations when multiple encryption modes coexist. + - module: The encryption module, formatted as "encryption module" + "#" + "encryption function." + - private_path: The path to the encryption key. If you provide a relative path, its root directory is `fate_flow/conf/`. + +## FateFlow Configuration +```yaml +host: 127.0.0.1 +http_port: 9380 +grpc_port: 9360 +proxy_name: rollsite +nginx: + host: + http_port: + grpc_port: +``` +- host: Host address. +- http_port: HTTP port number. +- grpc_port: gRPC port number. +- proxy_name: Command channel service name, supporting osx/rollsite/nginx. Detailed configurations need to be set within [Communication Engine Pool](#communication-engine-pool). +- nginx: Proxy service configuration for load balancing. + +## Database Configuration +```yaml +engine: sqlite +decrypt_key: +mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 +sqlite: + path: +``` +- engine: Database engine name. If set to "mysql" here, update the detailed MySQL configuration. +- decrypt_key: Encryption module, selected from [Encryption Module](#encryption-module). If not configured, it's considered to not use password encryption. If used, you need to set the "passwd" below to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- mysql: MySQL service configuration. If using password encryption functionality, set the "passwd" in this configuration to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- sqlite: SQLite file path, default path is `fate_flow/fate_flow_sqlite.db`. + +## Engine Configuration +```yaml +default_engines: + computing: standalone + federation: standalone + storage: standalone +``` + +- computing: Computing engine, supports "standalone," "eggroll," "spark." +- federation: Communication engine, supports "standalone," "rollsite," "osx," "rabbitmq," "pulsar." +- storage: Storage engine, supports "standalone," "eggroll," "hdfs." + +## Default Registered Algorithm Configuration +- name: Algorithm name. +- version: Algorithm version. If not configured, it uses the configuration in `fateflow.env`. +- device: Algorithm launch mode, local/docker/k8s, etc. + +## Communication Engine Pool +### Pulsar +```yaml +pulsar: + host: 192.168.0.5 + port: 6650 + mng_port: 8080 + cluster: standalone + tenant: fl-tenant + topic_ttl: 30 + route_table: + mode: replication + max_message_size: 1048576 +``` +### Nginx: +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + protocol: http +``` + +### RabbitMQ +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + protocol: http +``` + +### Rollsite +```yaml +rollsite: + host: 127.0.0.1 + port: 9370 +``` + +### OSx +```yaml + host: 127.0.0.1 + port: 9370 +``` + +## Computing Engine Pool +### Standalone +```yaml + cores: 32 +``` +- cores: Total resources. + +### Eggroll +```yaml +eggroll: + cores: 32 + nodes: 2 +``` +- cores: Total cluster resources. +- nodes: Number of node managers in the cluster. + +### Spark +```yaml +eggroll: + home: + cores: 32 +``` +- home: Spark home directory. If not filled, "pyspark" will be used as the computing engine. +- cores: Total resources. + +## Storage Engine Pool +```yaml + hdfs: + name_node: hdfs://fate-cluster +``` + +## Hook Module Configuration +```yaml +hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission +``` +- client_authentication: Client authentication hook. +- site_authentication: Site authentication hook. +- permission: Permission authentication hook. + +## Authentication Switch +```yaml +authentication: + client: false + site: false + permission: false +``` + +## Model Storage +```yaml +model_store: + engine: file + decrypt_key: + file: + path: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + tencent_cos: + Region: + SecretId: + SecretKey: + Bucket: +``` +- engine: Model storage engine, supports "file," "mysql", and "tencent_cos". +- decrypt_key: Encryption module, needs to be selected from [Encryption Module](#encryption-module). If not configured, it is assumed to not use password encryption. If used, you need to set the "passwd" below accordingly to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- file: Model storage directory, default location is `fate_flow/model`. +- mysql: MySQL service configuration; if using password encryption functionality, you need to set the "passwd" in this configuration to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- tencent_cos: Tencent Cloud key configuration. + +## ZooKeeper Configuration +```yaml +zookeeper: + hosts: + - 127.0.0.1:2181 + use_acl: true + user: fate + password: fate +``` \ No newline at end of file diff --git a/doc/system_conf.zh.md b/doc/system_conf.zh.md new file mode 100644 index 000000000..c0b4ef846 --- /dev/null +++ b/doc/system_conf.zh.md @@ -0,0 +1,221 @@ +# 系统配置描述文档 +FATE Flow使用yaml定义系统配置，配置路径位于: conf/service_conf.yaml, 具体配置内容及其含义如下： + +| 配置项 | 说明 | 值 | +|----------------------|------|------------------------------| +| party_id | 本方站点id | 如: "9999", "10000 | +| use_registry | 是否使用注册中心，当前仅支持zookeeper模式，需要保证zookeeper的配置正确；
注：若使用高可用模式，需保证该配置设置为true | true/false | +| encrypt | 加密模块 | 见[加密模块](#加密模块) | +| fateflow | FATE Flow服务的配置，主要包括端口、命令通道服务、代理等 | 见[FateFlow配置](#fateflow配置) | +| database | 数据库服务的配置信息 | 见[数据库配置](#数据库配置) | +| default_engines | 系统的引擎服务，主要包括计算、存储和通信引擎 | 见[引擎配置](#引擎配置) | +| default_provider | 组件的来源信息，主要包括提供方名称、组件版本和运行模式 | 见[默认注册算法配置](#默认注册算法配置) | +| federation | 通信服务池 | 见[通信引擎池](#通信引擎池) | +| computing | 计算服务池 | 见[计算引擎池](#计算引擎池) | +| storage | 存储服务池 | 见[存储引擎池](#存储配置) | +| hook_module | 钩子配置，当前支持客户端认证、站点端认证以及鉴权钩子 | 见[钩子模块配置](#钩子模块配置) | +| authentication | 认证&&鉴权开关 | 见[认证开关](#认证开关) | +| model_store | 模型存储配置 | 见[模型存储](#模型存储) | +| zookeeper | zookeeper服务的配置 | 见[zookeeper配置](#zookeeper配置) | + +## 加密模块 +```yaml +key_0: + module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt + private_path: private_key.pem +``` +该加密模块主要用于密码(如mysql密码)等内容加密： +- 其中"key_0"为加密模块的key(可以自定义名字)，便于其它配置中直接引用，多套加密模式共存。 + - module: 加密模块，拼接规则为：加密模块 + "#" + 加密函数。 + - private_path：密钥路径。如填相对路径，其根目录位于fate_flow/conf/ + +## FateFlow配置 +```yaml +host: 127.0.0.1 +http_port: 9380 +grpc_port: 9360 +proxy_name: rollsite +nginx: + host: + http_port: + grpc_port: +``` +- host: 主机地址; +- http_port：http端口号; +- grpc_port: grpc端口号; +- proxy_name: 命令通道服务名，支持osx/rollsite/nginx。详细配置需要在[通信引擎池](#通信引擎池) 里面配置; +- nginx: 代理服务配置，用于负载均衡。 + +## 数据库配置 +```yaml +engine: sqlite +decrypt_key: +mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 +sqlite: + path: +``` +- engine: 数据库引擎名字，如这里填"mysql"，则需要更新mysql的配置详细配置。 +- decrypt_key: 加密模块,需要从[加密模块](#加密模块)中选择。 若不配置，视为不使用密码加密；若使用，则需要将下面的passwd相应设置为密文。 +- mysql: mysql服务配置；若使用密码加密功能，需要将此配置中的"passwd"设置为密文，并在[加密模块](#加密模块)中配置密钥路径 +- sqlite: sqlite文件路径，默认路径为fate_flow/fate_flow_sqlite.db + +## 引擎配置 +```yaml +default_engines: + computing: standalone + federation: standalone + storage: standalone +``` + +- computing: 计算引擎，支持"standalone"、"eggroll"、"spark" +- federation: 通信引擎，支持"standalone"、"rollsite"、"osx"、"rabbitmq"、"pulsar" +- storage: 存储引擎，支持"standalone"、"eggroll"、"hdfs" + +## 默认注册算法配置 +- name: 算法名称 +- version: 算法版本，若不配置，则使用fateflow.env中的配置 +- device: 算法启动方式, local/docker/k8s等 + +## 通信引擎池 +### pulsar +```yaml +pulsar: + host: 192.168.0.5 + port: 6650 + mng_port: 8080 + cluster: standalone + tenant: fl-tenant + topic_ttl: 30 + # default conf/pulsar_route_table.yaml + route_table: + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 +``` +### nginx: +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + # http or grpc + protocol: http +``` + +### rabbitmq +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + # http or grpc + protocol: http +``` + +### rollsite +```yaml +rollsite: + host: 127.0.0.1 + port: 9370 +``` + +### osx +```yaml + host: 127.0.0.1 + port: 9370 +``` + +## 计算引擎池 +### standalone +```yaml + cores: 32 +``` +- cores: 资源总数 + +### eggroll +```yaml +eggroll: + cores: 32 + nodes: 2 +``` +- cores: 集群资源总数 +- nodes: 集群node-manager数量 + +### spark +```yaml +eggroll: + home: + cores: 32 +``` +- home: spark home目录，如果不填，将使用"pyspark"作为计算引擎。 +- cores: 资源总数 + +## 存储引擎池 +```yaml + hdfs: + name_node: hdfs://fate-cluster +``` + +## 钩子模块配置 +```yaml +hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission +``` +- client_authentication: 客户端认证钩子 +- site_authentication: 站点认证钩子 +- permission: 权限认证钩子 + +## 认证开关 +```yaml +authentication: + client: false + site: false + permission: false +``` + +## 模型存储 +```yaml +model_store: + engine: file + decrypt_key: + file: + path: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + tencent_cos: + Region: + SecretId: + SecretKey: + Bucket: +``` +- engine: 模型存储引擎，支持"file"、"mysql"和"tencent_cos"。 +- decrypt_key: 加密模块,需要从[加密模块](#加密模块)中选择。 若不配置，视为不使用密码加密；若使用，则需要将下面的passwd相应设置为密文。 +- file: 模型存储目录，默认位于： fate_flow/model +- mysql: mysql服务配置；若使用密码加密功能，需要将此配置中的"passwd"设置为密文，并在[加密模块](#加密模块)中配置密钥路径 +- tencent_cos: 腾讯云密钥配置 + + +## zookeeper配置 +```yaml +zookeeper: + hosts: + - 127.0.0.1:2181 + use_acl: true + user: fate + password: fate +``` \ No newline at end of file diff --git a/examples/lr/eggroll/lr_predict_dag.yaml b/examples/lr/eggroll/lr_predict_dag.yaml deleted file mode 100644 index b0a1f3f2d..000000000 --- a/examples/lr/eggroll/lr_predict_dag.yaml +++ /dev/null @@ -1,98 +0,0 @@ -dag: - conf: - model_id: 'xxx' - model_version: '0' - task_parallelism: 1 - scheduler_party_id: '9999' - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - - party_id: - - '10000' - role: arbiter - party_tasks: - guest_9999: - parties: - - party_id: - - '9999' - role: guest - tasks: - reader_1: - inputs: - parameters: - path: eggroll:///experiment/guest - format: raw_table - host_10000: - parties: - - party_id: - - '10000' - role: host - tasks: - reader_1: - inputs: - parameters: - path: eggroll:///experiment/host - format: raw_table - stage: predict - tasks: - intersection_0: - component_ref: intersection - dependent_tasks: - - reader_1 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: reader_1 - parameters: - method: raw - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default - lr_0: - component_ref: hetero_lr - conf: - backend: gpu - dependent_tasks: - - intersection_0 - inputs: - artifacts: - input_model: - model_warehouse: - output_artifact_key: output_model - producer_task: lr_0 - roles: - - guest - - host - test_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: intersection_0 - roles: - - guest - - host - parameters: - batch_size: 100 - learning_rate: 0.01 - max_iter: 1 - reader_1: - component_ref: reader - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default -schema_version: 2.0.0.alpha diff --git a/examples/lr/eggroll/lr_train_dag.yaml b/examples/lr/eggroll/lr_train_dag.yaml deleted file mode 100644 index 94eda0476..000000000 --- a/examples/lr/eggroll/lr_train_dag.yaml +++ /dev/null @@ -1,135 +0,0 @@ -dag: - conf: - task_parallelism: 1 - scheduler_party_id: '9999' - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - - party_id: - - '10000' - role: arbiter - party_tasks: - guest_9999: - conf: - test_reader_guest: 2 - parties: - - party_id: - - '9999' - role: guest - tasks: - reader_0: - inputs: - parameters: - path: eggroll:///experiment/guest - format: raw_table - host_10000: - conf: - test_reader_guest: 2 - parties: - - party_id: - - '10000' - role: host - tasks: - reader_0: - inputs: - parameters: - path: eggroll:///experiment/host - format: raw_table - stage: train - tasks: - evaluation_0: - component_ref: evaluation - dependent_tasks: - - lr_0 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: train_output_data - producer_task: lr_0 - roles: - - guest - parties: - - party_id: - - '9999' - role: guest - stage: default - intersection_0: - component_ref: intersection - dependent_tasks: - - reader_0 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: reader_0 - parameters: - method: raw - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default - intersection_1: - component_ref: intersection - dependent_tasks: - - reader_0 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: reader_0 - parameters: - method: raw - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default - lr_0: - component_ref: hetero_lr - dependent_tasks: - - intersection_0 - inputs: - artifacts: - train_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: intersection_0 - roles: - - guest - - host - validate_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: intersection_0 - roles: - - guest - - host - parameters: - batch_size: 100 - learning_rate: 0.01 - max_iter: 1 - reader_0: - component_ref: reader - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default -schema_version: 2.0.0.alpha diff --git a/examples/lr/predict_lr.yaml b/examples/lr/predict_lr.yaml new file mode 100644 index 000000000..fe3ed16d9 --- /dev/null +++ b/examples/lr/predict_lr.yaml @@ -0,0 +1,212 @@ +dag: + conf: + model_warehouse: + model_id: '202309081631313722080' + model_version: '0' + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + - party_id: + - '9998' + role: arbiter + party_tasks: + guest_9999: + parties: + - party_id: + - '9999' + role: guest + tasks: + psi_0: + inputs: + data: + input_data: + data_warehouse: + name: breast_hetero_guest + namespace: experiment + roles: + - guest + host_9998: + parties: + - party_id: + - '9998' + role: host + tasks: + psi_0: + inputs: + data: + input_data: + data_warehouse: + name: breast_hetero_host + namespace: experiment + roles: + - host + stage: predict + tasks: + binning_0: + component_ref: hetero_feature_binning + dependent_tasks: + - scale_0 + inputs: + data: + test_data: + task_output_artifact: + output_artifact_key: test_output_data + producer_task: scale_0 + model: + input_model: + model_warehouse: + output_artifact_key: output_model + producer_task: binning_0 + parameters: + adjustment_factor: 0.5 + bin_col: null + bin_idx: null + category_col: null + category_idx: null + local_only: false + method: quantile + n_bins: 10 + relative_error: 1.0e-06 + skip_metrics: false + split_pt_dict: null + transform_method: null + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + evaluation_0: + component_ref: evaluation + dependent_tasks: + - lr_0 + inputs: + data: + input_data: + task_output_artifact: + - output_artifact_key: test_output_data + producer_task: lr_0 + roles: + - guest + parameters: + default_eval_setting: binary + label_column_name: null + metrics: null + predict_column_name: null + parties: + - party_id: + - '9999' + role: guest + stage: default + lr_0: + component_ref: coordinated_lr + dependent_tasks: + - selection_0 + inputs: + data: + test_data: + task_output_artifact: + output_artifact_key: test_output_data + producer_task: selection_0 + roles: + - guest + - host + model: + input_model: + model_warehouse: + output_artifact_key: output_model + producer_task: lr_0 + roles: + - guest + - host + parameters: + batch_size: null + early_stop: diff + epochs: 10 + floating_point_precision: 23 + output_cv_data: true + threshold: 0.5 + tol: 0.0001 + psi_0: + component_ref: psi + inputs: + data: {} + parameters: {} + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + stage: default + scale_0: + component_ref: feature_scale + dependent_tasks: + - psi_0 + inputs: + data: + test_data: + task_output_artifact: + output_artifact_key: output_data + producer_task: psi_0 + model: + input_model: + model_warehouse: + output_artifact_key: output_model + producer_task: scale_0 + parameters: + feature_range: null + method: standard + scale_col: null + scale_idx: null + strict_range: true + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + selection_0: + component_ref: hetero_feature_selection + dependent_tasks: + - scale_0 + inputs: + data: + test_data: + task_output_artifact: + output_artifact_key: test_output_data + producer_task: scale_0 + model: + input_model: + model_warehouse: + output_artifact_key: train_output_model + producer_task: selection_0 + parameters: + iv_param: + filter_type: threshold + metrics: iv + threshold: 0.1 + keep_one: true + manual_param: null + method: + - iv + select_col: null + statistic_param: null + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host +schema_version: 2.0.0.beta diff --git a/examples/lr/standalone/lr_predict_dag.yaml b/examples/lr/standalone/lr_predict_dag.yaml deleted file mode 100644 index 336444362..000000000 --- a/examples/lr/standalone/lr_predict_dag.yaml +++ /dev/null @@ -1,108 +0,0 @@ -dag: - conf: - model_id: 'xxx' - model_version: '0' - task_parallelism: 1 - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - - party_id: - - '10001' - role: arbiter - party_tasks: - guest_9999: - parties: - - party_id: - - '9999' - role: guest - tasks: - reader_1: - inputs: - parameters: - delimiter: ',' - dtype: float32 - format: csv - id_name: id - label_name: y - label_type: float32 - path: file:///data/projects/fate/fateflow/examples/data/breast_hetero_guest.csv - host_10000: - parties: - - party_id: - - '10000' - role: host - tasks: - reader_1: - inputs: - parameters: - delimiter: ',' - dtype: float32 - format: csv - id_name: id - label_name: null - path: file:///data/projects/fate/fateflow/examples/data/breast_hetero_host.csv - scheduler_party_id: '10001' - stage: predict - tasks: - intersection_0: - component_ref: intersection - dependent_tasks: - - reader_1 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: reader_1 - parameters: - method: raw - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default - lr_0: - component_ref: hetero_lr - conf: - backend: gpu - dependent_tasks: - - intersection_0 - inputs: - artifacts: - input_model: - model_warehouse: - output_artifact_key: output_model - producer_task: lr_0 - roles: - - guest - - host - test_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: intersection_0 - roles: - - guest - - host - parameters: - batch_size: 100 - learning_rate: 0.01 - max_iter: 1 - reader_1: - component_ref: reader - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default -schema_version: 2.0.0.alpha - diff --git a/examples/lr/standalone/lr_train_dag.yaml b/examples/lr/standalone/lr_train_dag.yaml deleted file mode 100644 index 18661eb18..000000000 --- a/examples/lr/standalone/lr_train_dag.yaml +++ /dev/null @@ -1,145 +0,0 @@ -dag: - conf: - task_parallelism: 1 - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - - party_id: - - '10001' - role: arbiter - party_tasks: - guest_9999: - conf: - test_reader_guest: 2 - parties: - - party_id: - - '9999' - role: guest - tasks: - reader_0: - inputs: - parameters: - delimiter: ',' - dtype: float32 - format: csv - id_name: id - label_name: y - label_type: float32 - path: file:///data/projects/fate/fateflow/examples/data/breast_hetero_guest.csv - host_10000: - conf: - test_reader_guest: 2 - parties: - - party_id: - - '10000' - role: host - tasks: - reader_0: - inputs: - parameters: - delimiter: ',' - dtype: float32 - format: csv - id_name: id - label_name: null - path: file:///data/projects/fate/fateflow/examples/data/breast_hetero_host.csv - scheduler_party_id: '10001' - stage: train - tasks: - evaluation_0: - component_ref: evaluation - dependent_tasks: - - lr_0 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: train_output_data - producer_task: lr_0 - roles: - - guest - parties: - - party_id: - - '9999' - role: guest - stage: default - intersection_0: - component_ref: intersection - dependent_tasks: - - reader_0 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: reader_0 - parameters: - method: raw - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default - intersection_1: - component_ref: intersection - dependent_tasks: - - reader_0 - inputs: - artifacts: - input_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: reader_0 - parameters: - method: raw - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default - lr_0: - component_ref: hetero_lr - dependent_tasks: - - intersection_0 - inputs: - artifacts: - train_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: intersection_0 - roles: - - guest - - host - validate_data: - task_output_artifact: - output_artifact_key: output_data - producer_task: intersection_0 - roles: - - guest - - host - parameters: - batch_size: 100 - learning_rate: 0.01 - max_iter: 1 - reader_0: - component_ref: reader - parties: - - party_id: - - '9999' - role: guest - - party_id: - - '10000' - role: host - stage: default -schema_version: 2.0.0.alpha - diff --git a/examples/lr/train_lr.yaml b/examples/lr/train_lr.yaml new file mode 100644 index 000000000..b7d0384b4 --- /dev/null +++ b/examples/lr/train_lr.yaml @@ -0,0 +1,194 @@ +dag: + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + - party_id: + - '9998' + role: arbiter + party_tasks: + guest_9999: + parties: + - party_id: + - '9999' + role: guest + tasks: + psi_0: + inputs: + data: + input_data: + data_warehouse: + name: breast_hetero_guest + namespace: experiment + roles: + - guest + host_9998: + parties: + - party_id: + - '9998' + role: host + tasks: + psi_0: + inputs: + data: + input_data: + data_warehouse: + name: breast_hetero_host + namespace: experiment + roles: + - host + stage: train + tasks: + binning_0: + component_ref: hetero_feature_binning + dependent_tasks: + - scale_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_output_data + producer_task: scale_0 + model: {} + parameters: + adjustment_factor: 0.5 + bin_col: null + bin_idx: null + category_col: null + category_idx: null + local_only: false + method: quantile + n_bins: 10 + relative_error: 1.0e-06 + skip_metrics: false + split_pt_dict: null + transform_method: null + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + evaluation_0: + component_ref: evaluation + dependent_tasks: + - lr_0 + inputs: + data: + input_data: + task_output_artifact: + - output_artifact_key: train_output_data + producer_task: lr_0 + roles: + - guest + parameters: + default_eval_setting: binary + label_column_name: null + metrics: null + predict_column_name: null + parties: + - party_id: + - '9999' + role: guest + stage: default + lr_0: + component_ref: coordinated_lr + dependent_tasks: + - selection_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_output_data + producer_task: selection_0 + roles: + - guest + - host + model: {} + parameters: + batch_size: null + early_stop: diff + epochs: 10 + floating_point_precision: 23 + output_cv_data: true + threshold: 0.5 + tol: 0.0001 + psi_0: + component_ref: psi + inputs: + data: {} + parameters: {} + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + stage: default + scale_0: + component_ref: feature_scale + dependent_tasks: + - psi_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: output_data + producer_task: psi_0 + model: {} + parameters: + feature_range: null + method: standard + scale_col: null + scale_idx: null + strict_range: true + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + selection_0: + component_ref: hetero_feature_selection + dependent_tasks: + - binning_0 + - scale_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_output_data + producer_task: scale_0 + model: + input_models: + task_output_artifact: + - output_artifact_key: output_model + producer_task: binning_0 + parameters: + iv_param: + filter_type: threshold + metrics: iv + threshold: 0.1 + keep_one: true + manual_param: null + method: + - iv + select_col: null + statistic_param: null + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host +schema_version: 2.0.0.beta diff --git a/examples/model/export.json b/examples/model/export.json new file mode 100644 index 000000000..5efb445af --- /dev/null +++ b/examples/model/export.json @@ -0,0 +1,7 @@ +{ + "model_id": "202307251326130289470", + "model_version": "0", + "role": "guest", + "party_id": "9999", + "path": "xxx/dir" +} \ No newline at end of file diff --git a/examples/model/import.json b/examples/model/import.json new file mode 100644 index 000000000..442302855 --- /dev/null +++ b/examples/model/import.json @@ -0,0 +1,5 @@ +{ + "model_id": "xxx", + "model_version": "xxx", + "path": "xxx" +} \ No newline at end of file diff --git a/examples/permission/delete.json b/examples/permission/delete.json new file mode 100644 index 000000000..2736324e3 --- /dev/null +++ b/examples/permission/delete.json @@ -0,0 +1,7 @@ +{ + "party_id": "9999", + "component": "reader", + "dataset": [{ + "name": "xxx", "namespace": "xxx" + }] +} \ No newline at end of file diff --git a/examples/permission/grant.json b/examples/permission/grant.json new file mode 100644 index 000000000..2736324e3 --- /dev/null +++ b/examples/permission/grant.json @@ -0,0 +1,7 @@ +{ + "party_id": "9999", + "component": "reader", + "dataset": [{ + "name": "xxx", "namespace": "xxx" + }] +} \ No newline at end of file diff --git a/examples/provider/register.json b/examples/provider/register.json new file mode 100644 index 000000000..f9919beab --- /dev/null +++ b/examples/provider/register.json @@ -0,0 +1,9 @@ +{ + "name": "fate", + "device": "local", + "version": "2.0.0", + "metadata": { + "path": "xxx", + "venv": "xxx" + } +} \ No newline at end of file diff --git a/examples/test/data.py b/examples/test/data.py deleted file mode 100644 index 7e7fca4f7..000000000 --- a/examples/test/data.py +++ /dev/null @@ -1,12 +0,0 @@ -import json - -import requests - -base = "http://127.0.0.1:9380/v2" - - -def upload_data(): - uri = "/data/upload" - conf = json.load(open("../upload/upload_guest.json", "r")) - response = requests.post(base+uri, json=conf) - print(response.text) diff --git a/examples/test/job.py b/examples/test/job.py deleted file mode 100644 index 5d6fcea68..000000000 --- a/examples/test/job.py +++ /dev/null @@ -1,35 +0,0 @@ -import requests -from ruamel import yaml - -base = "http://127.0.0.1:9380/v2" - - -def submit_job(): - uri = "/job/submit" - dag = yaml.safe_load(open("./../lr/standalone/lr_train_dag.yaml", "r")) - response = requests.post(base+uri, json={"dag_schema": dag}) - print(response.text) - - -def query_job(job_id): - uri = "/job/query" - response = requests.post(base+uri, json={"job_id": job_id}) - print(response.text) - - -def stop_job(job_id): - uri = "/job/stop" - response = requests.post(base+uri, json={"job_id": job_id}) - print(response.text) - - -def query_task(job_id, role, party_id, task_name): - uri = "/job/task/query" - response = requests.post(base+uri, json={"job_id": job_id, "role": role, "party_id": party_id, "task_name": task_name}) - print(response.text) - - -def rerun_job(job_id): - uri = "/job/rerun" - response = requests.post(base+uri, json={"job_id": job_id}) - print(response.text) \ No newline at end of file diff --git a/examples/test/output.py b/examples/test/output.py deleted file mode 100644 index 8ac0ed90c..000000000 --- a/examples/test/output.py +++ /dev/null @@ -1,39 +0,0 @@ -import requests - -base = "http://127.0.0.1:9380/v2" - - -def metric_key_query(job_id, role, party_id, task_name): - uri = "/output/metric/key/query" - data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "task_name": task_name - } - response = requests.get(base+uri, params=data) - print(response.text) - - -def metric_query(job_id, role, party_id, task_name): - uri = "/output/metric/query" - data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "task_name": task_name - } - response = requests.get(base+uri, params=data) - print(response.text) - - -def model_query(job_id, role, party_id, task_name): - uri = "/output/model/query" - data = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "task_name": task_name - } - response = requests.get(base+uri, params=data) - print(response.text) diff --git a/examples/test/site.py b/examples/test/site.py deleted file mode 100644 index 5ac377e90..000000000 --- a/examples/test/site.py +++ /dev/null @@ -1,9 +0,0 @@ -import requests - -base = "http://127.0.0.1:9380/v2" - - -def site_info_query(): - uri = "/site/info/query" - response = requests.get(base+uri) - print(response.text) diff --git a/examples/transformer/transformer_guest.json b/examples/transformer/transformer_guest.json new file mode 100644 index 000000000..7406d774f --- /dev/null +++ b/examples/transformer/transformer_guest.json @@ -0,0 +1,8 @@ +{ + "namespace": "experiment", + "name": "breast_hetero_guest", + "data_warehouse": { + "name": "75096508-400c-11ee-8646-16b977118319", + "namespace": "upload" + } +} diff --git a/examples/transformer/transformer_host.json b/examples/transformer/transformer_host.json new file mode 100644 index 000000000..645123fe2 --- /dev/null +++ b/examples/transformer/transformer_host.json @@ -0,0 +1,8 @@ +{ + "namespace": "experiment", + "name": "breast_hetero_host", + "data_warehouse": { + "name": "b47f68d6-400c-11ee-8646-16b977118319", + "namespace": "upload" + } +} diff --git a/examples/upload/upload_guest.json b/examples/upload/upload_guest.json index 45cfd27f7..68c57de04 100644 --- a/examples/upload/upload_guest.json +++ b/examples/upload/upload_guest.json @@ -1,15 +1,11 @@ { - "file": "/data/projects/fate/examples/data/breast_hetero_guest.csv", - "head": 1, - "partitions": 4, - "namespace": "experiment", - "name": "guest", - "storage_engine": "eggroll", - "destroy": 1, + "file": "examples/data/breast_hetero_guest.csv", + "head": true, + "partitions": 16, + "extend_sid": true, "meta": { "delimiter": ",", "label_name": "y", - "dtype": "float32", - "label_type": "float32" + "match_id_name": "id" } } diff --git a/examples/upload/upload_host.json b/examples/upload/upload_host.json index 7f8369094..0fb688987 100644 --- a/examples/upload/upload_host.json +++ b/examples/upload/upload_host.json @@ -1,13 +1,10 @@ { - "file": "/data/projects/fate/examples/data/breast_hetero_host.csv", - "head": 1, - "partitions": 4, - "namespace": "experiment", - "name": "host", - "storage_engine": "eggroll", - "destroy": 1, + "file": "examples/data/breast_hetero_host.csv", + "head": true, + "partitions": 16, + "extend_sid": true, "meta": { "delimiter": ",", - "dtype": "float32" + "match_id_name": "id" } } diff --git a/fateflow.env b/fateflow.env new file mode 100644 index 000000000..03d34cae3 --- /dev/null +++ b/fateflow.env @@ -0,0 +1,3 @@ +FATE=2.0.0.beta +FATEFlow=2.0.0.beta +PYTHON=3.8 \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 18d5dd94a..8b3f7a461 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,25 +9,9 @@ nav: - Home: index.md - Docs: #- ... | flat | *.md - - document_navigation.md - fate_flow.md - - fate_flow_data_access.md - - fate_flow_component_registry.md - - fate_flow_job_scheduling.md - - fate_flow_resource_management.md - - fate_flow_tracking.md - - fate_flow_monitoring.md - - fate_flow_model_registry.md - - fate_flow_authority_management.md - - fate_flow_permission_management.md - - fate_flow_server_operation.md - - fate_flow_service_registry.md - - fate_flow_model_migration.md - - fate_flow_client.md - - fate_flow_http_api_call_demo.md - - configuration_instruction.md - - system_operational.md - - faq.md + - quick_start.md + - system_conf.md - API: swagger/index.md theme: @@ -64,7 +48,13 @@ plugins: - i18n: default_language: en languages: - zh: 中文 + - locale: zh + name: 中文 + build: true + - locale: en + name: English + build: true + default: true - markdown-include-snippet: base_path: doc diff --git a/python/fate_flow/__init__.py b/python/fate_flow/__init__.py index ae946a49c..8d9ccccee 100644 --- a/python/fate_flow/__init__.py +++ b/python/fate_flow/__init__.py @@ -12,3 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from ._info import __provider__, __version__ diff --git a/python/fate_flow/_info.py b/python/fate_flow/_info.py new file mode 100644 index 000000000..2af52a733 --- /dev/null +++ b/python/fate_flow/_info.py @@ -0,0 +1,16 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +__version__ = "2.0.0-beta" +__provider__ = "fate_flow" diff --git a/python/fate_flow/apps/__init__.py b/python/fate_flow/apps/__init__.py index 514892590..c9dfac117 100644 --- a/python/fate_flow/apps/__init__.py +++ b/python/fate_flow/apps/__init__.py @@ -13,50 +13,133 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os.path import sys +import types +import typing as t + from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path -from flask import Blueprint, Flask +from flask import Blueprint, Flask, request from werkzeug.wrappers.request import Request -from fate_flow.settings import API_VERSION, getLogger -from fate_flow.utils.api_utils import args_error_response, server_error_response +from fate_flow.controller.app_controller import PermissionController +from fate_flow.entity.code import ReturnCode +from fate_flow.hook import HookManager +from fate_flow.hook.common.parameters import AuthenticationParameters +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import API_VERSION, CLIENT_AUTHENTICATION, SITE_AUTHENTICATION, \ + ADMIN_PAGE, PARTY_ID, INTERCONN_API_VERSION +from fate_flow.utils.api_utils import API from fate_flow.utils.base_utils import CustomJSONEncoder __all__ = ['app'] -logger = getLogger('flask.app') +app_list = ["client", "partner", "scheduler", "worker"] +interop_app_list = ["interop"] Request.json = property(lambda self: self.get_json(force=True, silent=True)) app = Flask(__name__) app.url_map.strict_slashes = False -app.errorhandler(422)(args_error_response) -app.errorhandler(Exception)(server_error_response) -app.json_encoder = CustomJSONEncoder +app.errorhandler(422)(API.Output.args_error_response) +app.errorhandler(Exception)(API.Output.server_error_response) +app.json_provider_class = CustomJSONEncoder + + +def search_pages_path(pages_dir): + return [path for path in pages_dir.glob('*_app.py') if not path.name.startswith('.')] -def register_page(page_path): - page_name = page_path.stem.rstrip('_app') +def register_page(page_path, func=None, prefix=API_VERSION): + page_name = page_path.stem.rstrip('app').rstrip("_") module_name = '.'.join(page_path.parts[page_path.parts.index('apps')-1:-1] + (page_name, )) spec = spec_from_file_location(module_name, page_path) page = module_from_spec(spec) page.app = app page.manager = Blueprint(page_name, module_name) + rule_methods_list = [] + + # rewrite blueprint route to get rule_list + def route(self, rule: str, **options: t.Any) -> t.Callable: + def decorator(f: t.Callable) -> t.Callable: + endpoint = options.pop("endpoint", None) + rule_methods_list.append((rule, options.get("methods", []))) + self.add_url_rule(rule, endpoint, f, **options) + return f + return decorator + + page.manager.route = types.MethodType(route, page.manager) + + if func: + page.manager.before_request(func) sys.modules[module_name] = page spec.loader.exec_module(page) - page_name = getattr(page, 'page_name', page_name) - url_prefix = f'/{API_VERSION}/{page_name}' - + url_prefix = f'/{prefix}/{page_name}' app.register_blueprint(page.manager, url_prefix=url_prefix) + return page_name, [(os.path.join(url_prefix, rule_methods[0].lstrip("/")), rule_methods[1]) for rule_methods in rule_methods_list] + + +def client_authentication_before_request(): + if CLIENT_AUTHENTICATION: + result = HookManager.client_authentication(AuthenticationParameters( + request.path, request.method, request.headers, + request.form, request.data, request.json, request.full_path + )) + + if result.code != ReturnCode.Base.SUCCESS: + return API.Output.json(result.code, result.message) + + +def site_authentication_before_request(): + if SITE_AUTHENTICATION: + result = HookManager.site_authentication(AuthenticationParameters( + request.path, request.method, request.headers, + request.form, request.data, request.json, request.full_path + )) + + if result.code != ReturnCode.Base.SUCCESS: + return API.Output.json(result.code, result.message) + + +def init_apps(): + urls_dict = {} + before_request_func = { + "client": client_authentication_before_request, + "partner": site_authentication_before_request, + "scheduler": site_authentication_before_request + } + for key in app_list: + urls_dict[key] = [register_page(path, before_request_func.get(key)) for path in search_pages_path(Path(__file__).parent / key)] + for key in interop_app_list: + urls_dict[key] = [register_page(path, prefix=INTERCONN_API_VERSION) for path in search_pages_path(Path(__file__).parent / key)] + if CLIENT_AUTHENTICATION or SITE_AUTHENTICATION: + _init_permission_group(urls=urls_dict) + +def _init_permission_group(urls: dict): + for role, role_items in urls.items(): + super_role = "super_" + role + if role in ["scheduler", "partner"]: + role = "site" + super_role = "site" + RuntimeConfig.set_client_roles(role, super_role) + for resource, rule_methods_list in role_items: + for rule_methods in rule_methods_list: + rule = rule_methods[0] + methods = rule_methods[1] + for method in methods: + if resource in ADMIN_PAGE: + PermissionController.add_policy(super_role, rule, method) + else: + PermissionController.add_policy(super_role, rule, method) + PermissionController.add_policy(role, rule, method) + PermissionController.add_role_for_user("admin", super_role, init=True) + PermissionController.add_role_for_user(PARTY_ID, "site", init=True) -for path in Path(__file__).parent.glob('*/*_app.py'): - if path.name.startswith(('.', '_')): - continue - register_page(path) +init_apps() diff --git a/python/fate_flow/apps/client/client_app.py b/python/fate_flow/apps/client/client_app.py new file mode 100644 index 000000000..7bbd4a080 --- /dev/null +++ b/python/fate_flow/apps/client/client_app.py @@ -0,0 +1,91 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import APP_NAME, APP_ID, PARTY_ID, SITE_APP_ID, SITE_APP_TOKEN +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.types import AppType +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.runtime.system_settings import APP_MANAGER_PAGE +from fate_flow.utils.api_utils import API + +page_name = APP_MANAGER_PAGE + + +@manager.route('/client/create', methods=['POST']) +@API.Input.json(app_name=fields.String(required=True), desc=APP_NAME) +def create_client_app(app_name): + data = AppManager.create_app(app_name=app_name, app_type=AppType.CLIENT, init=False) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=data) + + +@manager.route('/client/delete', methods=['POST']) +@API.Input.json(app_id=fields.String(required=True), desc=APP_ID) +def delete_client_app(app_id): + status = AppManager.delete_app(app_id=app_id, app_type=AppType.CLIENT, init=False) + return API.Output.json(data={"status": status}) + + +@manager.route('/client/query', methods=['GET']) +@API.Input.params(app_id=fields.String(required=False), desc=APP_ID) +@API.Input.params(app_name=fields.String(required=False), desc=APP_NAME) +def query_client_app(app_id=None, app_name=None): + apps = AppManager.query_app(app_id=app_id, app_name=app_name, app_type=AppType.CLIENT) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=[app.to_human_model_dict() for app in apps]) + + +@manager.route('/site/create', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +def create_site_app(party_id): + data = AppManager.create_app(app_name=party_id, app_type=AppType.SITE) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=data) + + +@manager.route('/site/delete', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +def delete_site_app(party_id): + status = AppManager.delete_app(app_name=party_id, app_type=AppType.SITE, init=True) + return API.Output.json(data={"status": status}) + + +@manager.route('/site/query', methods=['GET']) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +def query_site_app(party_id=None): + apps = AppManager.query_app(app_name=party_id, app_type=AppType.SITE,init=True) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=[app.to_human_model_dict() for app in apps]) + + +@manager.route('/partner/create', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(app_id=fields.String(required=True), desc=SITE_APP_ID) +@API.Input.json(app_token=fields.String(required=True), desc=SITE_APP_TOKEN) +def create_partner_app(party_id, app_id, app_token): + data = AppManager.create_partner_app(app_id=app_id, party_id=party_id, app_token=app_token) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=data) + + +@manager.route('/partner/delete', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +def delete_partner_app(party_id): + status = AppManager.delete_partner_app(party_id=party_id, init=False) + return API.Output.json(data={"status": status}) + + +@manager.route('/partner/query', methods=['GET']) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +def query_partner_app(party_id=None): + apps = AppManager.query_partner_app(party_id=party_id) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=[app.to_human_model_dict() for app in apps]) diff --git a/python/fate_flow/apps/client/data_app.py b/python/fate_flow/apps/client/data_app.py index 516ad261a..6a7fa596f 100644 --- a/python/fate_flow/apps/client/data_app.py +++ b/python/fate_flow/apps/client/data_app.py @@ -15,25 +15,65 @@ # from webargs import fields -from fate_flow.entity.types import ReturnCode -from fate_flow.utils.api_utils import get_json_result, validate_request_json, validate_request_params -from fate_flow.utils.data_upload import Upload, UploadParam +from fate_flow.apps.desc import SERVER_FILE_PATH, HEAD, PARTITIONS, META, EXTEND_SID, NAMESPACE, NAME, DATA_WAREHOUSE, \ + DROP, SITE_NAME +from fate_flow.engine import storage +from fate_flow.manager.components.component_manager import ComponentManager +from fate_flow.manager.data.data_manager import DataManager +from fate_flow.utils.api_utils import API +from fate_flow.errors.server_error import NoFoundTable page_name = "data" -@manager.route('/upload', methods=['POST']) -@validate_request_json(file=fields.String(required=True), head=fields.Bool(required=True), - namespace=fields.String(required=True), name=fields.String(required=True), - partitions=fields.Integer(required=True), storage_engine=fields.String(required=False), - destroy=fields.Bool(required=False), meta=fields.Dict(required=True)) -def upload_data(file, head, partitions, namespace, name, meta, destroy=False, storage_engine=""): - data = Upload().run(parameters=UploadParam(file=file, head=head, partitions=partitions, namespace=namespace, - name=name, storage_engine=storage_engine, meta=meta, destroy=destroy)) - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success", data=data) +@manager.route('/component/upload', methods=['POST']) +@API.Input.json(file=fields.String(required=True), desc=SERVER_FILE_PATH) +@API.Input.json(head=fields.Bool(required=True), desc=HEAD) +@API.Input.json(partitions=fields.Integer(required=True), desc=PARTITIONS) +@API.Input.json(meta=fields.Dict(required=True), desc=META) +@API.Input.json(extend_sid=fields.Bool(required=False), desc=EXTEND_SID) +@API.Input.json(namespace=fields.String(required=False), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=False), desc=NAME) +def upload_data(file, head, partitions, meta, namespace=None, name=None, extend_sid=False): + result = ComponentManager.upload( + file=file, head=head, partitions=partitions, meta=meta, namespace=namespace, name=name, extend_sid=extend_sid + ) + return API.Output.json(**result) + + +@manager.route('/component/download', methods=['POST']) +@API.Input.json(name=fields.String(required=True), desc=NAME) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(path=fields.String(required=False), desc=SERVER_FILE_PATH) +def download_data(namespace, name, path): + result = ComponentManager.download( + path=path, namespace=namespace, name=name + ) + return API.Output.json(**result) + + +@manager.route('/component/dataframe/transformer', methods=['POST']) +@API.Input.json(data_warehouse=fields.Dict(required=True), desc=DATA_WAREHOUSE) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=True), desc=NAME) +@API.Input.json(site_name=fields.String(required=False), desc=SITE_NAME) +@API.Input.json(drop=fields.Bool(required=False), desc=DROP) +def transformer_data(data_warehouse, namespace, name, drop=True, site_name=None): + result = ComponentManager.dataframe_transformer(data_warehouse, namespace, name, drop, site_name) + return API.Output.json(**result) @manager.route('/download', methods=['GET']) -@validate_request_params(name=fields.String(required=True), namespace=fields.String(required=True)) -def download(name, namespace): - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success") +@API.Input.params(name=fields.String(required=True), desc=NAME) +@API.Input.params(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.params(header=fields.String(required=False), desc=HEAD) +def download(namespace, name, header=None): + data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) + if not data_table_meta: + raise NoFoundTable(name=name, namespace=namespace) + return DataManager.send_table( + output_tables_meta={"data": data_table_meta}, + tar_file_name=f'download_data_{namespace}_{name}.tar.gz', + need_head=header + ) + diff --git a/python/fate_flow/apps/client/job_app.py b/python/fate_flow/apps/client/job_app.py index 505fcf529..1e1b7f94a 100644 --- a/python/fate_flow/apps/client/job_app.py +++ b/python/fate_flow/apps/client/job_app.py @@ -13,57 +13,166 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import io +import os +import tarfile + from webargs import fields +from fate_flow.apps.desc import DAG_SCHEMA, USER_NAME, JOB_ID, ROLE, PARTY_ID, STATUS, LIMIT, PAGE, PARTNER, ORDER_BY, \ + ORDER, DESCRIPTION, TASK_NAME, TASK_ID, TASK_VERSION, NODES from fate_flow.controller.job_controller import JobController -from fate_flow.entity.types import ReturnCode -from fate_flow.utils.api_utils import get_json_result, validate_request_json +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundJob, NoFoundTask, FileNoFound +from fate_flow.utils import job_utils +from fate_flow.utils.api_utils import API +from fate_flow.manager.pipeline import pipeline as pipeline_manager @manager.route('/submit', methods=['POST']) -@validate_request_json(dag_schema=fields.Dict(required=True)) -def submit_job(dag_schema): - submit_result = JobController.request_create_job(dag_schema) - return get_json_result(**submit_result) +@API.Input.json(dag_schema=fields.Dict(required=True), desc=DAG_SCHEMA) +@API.Input.headers(user_name=fields.String(required=False), desc=USER_NAME) +def submit_job(dag_schema, user_name=None): + submit_result = JobController.request_create_job(dag_schema, user_name) + return API.Output.json(**submit_result) -@manager.route('/query', methods=['POST']) -@validate_request_json(job_id=fields.String(required=False), role=fields.String(required=False), - party_id=fields.String(required=False), status=fields.String(required=False)) -def query_job(job_id=None, role=None, party_id=None, status=None): - jobs = JobController.query_job(job_id=job_id, role=role, party_id=party_id, status=status) +@manager.route('/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False), desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(status=fields.String(required=False), desc=STATUS) +@API.Input.headers(user_name=fields.String(required=False), desc=USER_NAME) +def query_job(job_id=None, role=None, party_id=None, status=None, user_name=None): + jobs = JobController.query_job(job_id=job_id, role=role, party_id=party_id, status=status, user_name=user_name) if not jobs: - return get_json_result(code=ReturnCode.Job.NOT_FOUND, message="job no found") - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success", - data=[job.to_human_model_dict() for job in jobs]) - - -@manager.route('/task/query', methods=['POST']) -@validate_request_json(job_id=fields.String(required=False), role=fields.String(required=False), - party_id=fields.String(required=False), status=fields.String(required=False), - task_name=fields.String(required=False), task_id=fields.String(required=False), - task_version=fields.Integer(required=False)) -def query_task(job_id=None, role=None, party_id=None, status=None, task_name=None, task_id=None, task_version=None): - tasks = JobController.query_tasks(job_id=job_id, role=role, party_id=party_id, status=status, task_name=task_name, - task_id=task_id, task_version=task_version) - if not tasks: - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task no found") - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success", - data=[task.to_human_model_dict() for task in tasks]) + return API.Output.fate_flow_exception(NoFoundJob(job_id=job_id, role=role, party_id=party_id, status=status)) + return API.Output.json(data=[job.to_human_model_dict() for job in jobs]) @manager.route('/stop', methods=['POST']) -@validate_request_json(job_id=fields.String(required=True)) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) def request_stop_job(job_id=None): stop_result = JobController.request_stop_job(job_id=job_id) - return get_json_result(**stop_result) + return API.Output.json(**stop_result) @manager.route('/rerun', methods=['POST']) -@validate_request_json(job_id=fields.String(required=True)) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) def request_rerun_job(job_id=None): jobs = JobController.query_job(job_id=job_id) if not jobs: - return get_json_result(code=ReturnCode.Job.NOT_FOUND, message="job not found") + return API.Output.fate_flow_exception(NoFoundJob(job_id=job_id)) rerun_result = JobController.request_rerun_job(job=jobs[0]) - return get_json_result(**rerun_result) + return API.Output.json(**rerun_result) + + +@manager.route('/list/query', methods=['GET']) +@API.Input.params(limit=fields.Integer(required=False), desc=LIMIT) +@API.Input.params(page=fields.Integer(required=False), desc=PAGE) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(description=fields.String(required=False), desc=DESCRIPTION) +@API.Input.params(partner=fields.String(required=False), desc=PARTNER) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(role=fields.List(fields.Str(), required=False), desc=ROLE) +@API.Input.params(status=fields.List(fields.Str(), required=False), desc=STATUS) +@API.Input.params(order_by=fields.String(required=False), desc=ORDER_BY) +@API.Input.params(order=fields.String(required=False), desc=ORDER) +@API.Input.headers(user_name=fields.String(required=False), desc=USER_NAME) +def query_job_list(limit=0, page=0, job_id=None, description=None, partner=None, party_id=None, role=None, status=None, + order_by=None, order=None, user_name=None): + count, data = JobController.query_job_list( + limit, page, job_id, description, partner, party_id, role, status, order_by, order, user_name + ) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", + data={"count": count, "data": data}) + + +@manager.route('/task/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False),desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(status=fields.String(required=False), desc=STATUS) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(task_id=fields.String(required=False), desc=TASK_ID) +@API.Input.params(task_version=fields.Integer(required=False), desc=TASK_VERSION) +def query_task(job_id=None, role=None, party_id=None, status=None, task_name=None, task_id=None, task_version=None): + tasks = JobController.query_tasks(job_id=job_id, role=role, party_id=party_id, status=status, task_name=task_name, + task_id=task_id, task_version=task_version) + if not tasks: + return API.Output.fate_flow_exception(NoFoundTask()) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", + data=[task.to_human_model_dict() for task in tasks]) + + +@manager.route('/task/list/query', methods=['GET']) +@API.Input.params(limit=fields.Integer(required=False), desc=LIMIT) +@API.Input.params(page=fields.Integer(required=False), desc=PAGE) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False), desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(order_by=fields.String(required=False), desc=ORDER_BY) +@API.Input.params(order=fields.String(required=False), desc=ORDER) +def query_task_list(limit=0, page=0, job_id=None, role=None, party_id=None, task_name=None, order_by=None, order=None): + count, data = JobController.query_task_list( + limit, page, job_id, role, party_id, task_name, order_by, order + ) + return API.Output.json( + code=ReturnCode.Base.SUCCESS, message="success", + data={"count": count, "data": data} + ) + + +@manager.route('/log/download', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +def download_job_logs(job_id): + job_log_dir = job_utils.get_job_log_directory(job_id=job_id) + if not os.path.exists(job_log_dir): + return API.Output.fate_flow_exception(e=FileNoFound(path=job_log_dir)) + memory_file = io.BytesIO() + with tarfile.open(fileobj=memory_file, mode='w:gz') as tar: + for root, _, files in os.walk(job_log_dir): + for file in files: + full_path = os.path.join(root, file) + rel_path = os.path.relpath(full_path, job_log_dir) + tar.add(full_path, rel_path) + memory_file.seek(0) + return API.Output.file( + memory_file, attachment_filename=f'job_{job_id}_log.tar.gz', as_attachment=True, mimetype="application/gzip" + ) + + +@manager.route('/queue/clean', methods=['POST']) +def clean_queue(): + data = JobController.clean_queue() + return API.Output.json(data=data) + + +@manager.route('/clean', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +def clean_job(job_id): + JobController.clean_job(job_id=job_id) + return API.Output.json() + + +@manager.route('/notes/add', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(notes=fields.String(required=True), desc=NODES) +def add_notes(job_id, role, party_id, notes): + JobController.add_notes(job_id=job_id, role=role, party_id=party_id, notes=notes) + return API.Output.json() + + +@manager.route('/dag/dependency', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +def dag_dependency(job_id, role, party_id): + jobs = JobController.query_job(job_id=job_id, role=role, party_id=party_id) + if not jobs: + return API.Output.fate_flow_exception(NoFoundJob(job_id=job_id)) + data = pipeline_manager.pipeline_dag_dependency(jobs[0]) + return API.Output.json(data=data) diff --git a/python/fate_flow/apps/client/log_app.py b/python/fate_flow/apps/client/log_app.py new file mode 100644 index 000000000..23ecfd753 --- /dev/null +++ b/python/fate_flow/apps/client/log_app.py @@ -0,0 +1,49 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import LOG_TYPE, JOB_ID, ROLE, PARTY_ID, TASK_NAME, INSTANCE_ID, BEGIN, END +from fate_flow.manager.log.log_manager import LogManager +from fate_flow.utils.api_utils import API +from fate_flow.utils.wraps_utils import cluster_route + + +@manager.route('/count', methods=['GET']) +@API.Input.params(log_type=fields.String(required=True), desc=LOG_TYPE) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False), desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(instance_id=fields.String(required=False), desc=INSTANCE_ID) +@cluster_route +def count(log_type, job_id, role=None, party_id=None, task_name=None, instance_id=None): + data = LogManager(log_type, job_id, role=role, party_id=party_id, task_name=task_name).count() + return API.Output.json(data=data) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(log_type=fields.String(required=True), desc=LOG_TYPE) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(begin=fields.Integer(required=False), desc=BEGIN) +@API.Input.params(end=fields.Integer(required=False), desc=END) +@API.Input.params(instance_id=fields.String(required=False), desc=INSTANCE_ID) +@cluster_route +def get(log_type, job_id, role, party_id, task_name=None, begin=None, end=None, instance_id=None): + data = LogManager(log_type, job_id, role=role, party_id=party_id, task_name=task_name).cat_log(begin=begin, end=end) + return API.Output.json(data=data) diff --git a/python/fate_flow/apps/client/model_app.py b/python/fate_flow/apps/client/model_app.py new file mode 100644 index 000000000..5b940515b --- /dev/null +++ b/python/fate_flow/apps/client/model_app.py @@ -0,0 +1,99 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os.path +from tempfile import TemporaryDirectory + +from flask import request +from webargs import fields + +from fate_flow.apps.desc import MODEL_ID, MODEL_VERSION, PARTY_ID, ROLE, SERVER_DIR_PATH, TASK_NAME, OUTPUT_KEY +from fate_flow.errors.server_error import NoFoundFile +from fate_flow.manager.model.model_manager import PipelinedModel +from fate_flow.utils.api_utils import API + + +@manager.route('/load', methods=['POST']) +def load(): + # todo: + return API.Output.json() + + +@manager.route('/migrate', methods=['POST']) +def migrate(): + # todo: + return API.Output.json() + + +@manager.route('/export', methods=['POST']) +@API.Input.json(model_id=fields.String(required=True), desc=MODEL_ID) +@API.Input.json(model_version=fields.String(required=True), desc=MODEL_VERSION) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(path=fields.String(required=True), desc=SERVER_DIR_PATH) +def export(model_id, model_version, party_id, role, path): + file_list = PipelinedModel.export_model( + model_id=model_id, + model_version=model_version, + party_id=party_id, + role=role, + dir_path=path + ) + return API.Output.json(data=file_list) + + +@manager.route('/import', methods=['POST']) +@API.Input.form(model_id=fields.String(required=True), desc=MODEL_ID) +@API.Input.form(model_version=fields.String(required=True), desc=MODEL_VERSION) +def import_model(model_id, model_version): + file = request.files.get('file') + if not file: + raise NoFoundFile() + with TemporaryDirectory() as temp_dir: + path = os.path.join(temp_dir, file.name) + file.save(path) + PipelinedModel.import_model(model_id, model_version, path, temp_dir) + return API.Output.json() + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(model_id=fields.String(required=True), desc=MODEL_ID) +@API.Input.json(model_version=fields.String(required=True), desc=MODEL_VERSION) +@API.Input.json(role=fields.String(required=False), desc=ROLE) +@API.Input.json(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.json(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.json(output_key=fields.String(required=False), desc=OUTPUT_KEY) +def delete_model(model_id, model_version, role=None, party_id=None, task_name=None, output_key=None): + count = PipelinedModel.delete_model( + model_id=model_id, + model_version=model_version, + party_id=party_id, + role=role, + task_name=task_name, + output_key=output_key + ) + return API.Output.json(data={"count": count}) + + +@manager.route('/store', methods=['POST']) +def store(): + # todo: + return API.Output.json() + + +@manager.route('/restore', methods=['POST']) +def restore(): + # todo: + return API.Output.json() diff --git a/python/fate_flow/apps/client/output_app.py b/python/fate_flow/apps/client/output_app.py index 54eb815a6..1f615c09d 100644 --- a/python/fate_flow/apps/client/output_app.py +++ b/python/fate_flow/apps/client/output_app.py @@ -15,45 +15,175 @@ # from webargs import fields -from fate_flow.entity.types import ReturnCode -from fate_flow.manager.model_manager import PipelinedModel -from fate_flow.manager.output_manager import OutputMetric +from fate_flow.apps.desc import JOB_ID, ROLE, PARTY_ID, TASK_NAME, FILTERS, OUTPUT_KEY +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundTask +from fate_flow.manager.data.data_manager import DataManager +from fate_flow.manager.model.model_manager import PipelinedModel +from fate_flow.manager.metric.metric_manager import OutputMetric from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.api_utils import get_json_result, validate_request_params +from fate_flow.utils.api_utils import API @manager.route('/metric/key/query', methods=['GET']) -@validate_request_params(job_id=fields.String(required=True), role=fields.String(required=True), - party_id=fields.String(required=True), task_name=fields.String(required=True)) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) def query_metric_key(job_id, role, party_id, task_name): tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) if not tasks: - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task not found") + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) metric_keys = OutputMetric(job_id=job_id, role=role, party_id=party_id, task_name=task_name, task_id=tasks[0].f_task_id, task_version=tasks[0].f_task_version).query_metric_keys() - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success', data=metric_keys) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success', data=metric_keys) @manager.route('/metric/query', methods=['GET']) -@validate_request_params(job_id=fields.String(required=True), role=fields.String(required=True), - party_id=fields.String(required=True), task_name=fields.String(required=True), - filters=fields.Dict(required=False)) +@API.Input.params(job_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +@API.Input.params(filters=fields.Dict(required=False), desc=FILTERS) def query_metric(job_id, role, party_id, task_name, filters=None): tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) if not tasks: - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task not found") + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) metrics = OutputMetric(job_id=job_id, role=role, party_id=party_id, task_name=task_name, task_id=tasks[0].f_task_id, task_version=tasks[0].f_task_version).read_metrics(filters) - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success', data=metrics) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success', data=metrics) + + +@manager.route('/metric/delete', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(task_name=fields.String(required=True), desc=TASK_NAME) +def delete_metric(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + metric_keys = OutputMetric( + job_id=job_id, role=role, party_id=party_id, task_name=task_name, + task_id=tasks[0].f_task_id, task_version=tasks[0].f_task_version + ).delete_metrics() + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success', data=metric_keys) @manager.route('/model/query', methods=['GET']) -@validate_request_params(job_id=fields.String(required=True), role=fields.String(required=True), - party_id=fields.String(required=True), task_name=fields.String(required=True)) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) def query_model(job_id, role, party_id, task_name): tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) if not tasks: - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task not found") + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + model_data = PipelinedModel.read_model(task.f_job_id, task.f_role, task.f_party_id, task.f_task_name) + return API.Output.json(data=model_data) + + +@manager.route('/model/download', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def download(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return PipelinedModel.download_model(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, + task_name=task.f_task_name) + + +@manager.route('/model/delete', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(task_name=fields.String(required=True), desc=TASK_NAME) +def delete_model(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + PipelinedModel.delete_model( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name) + return API.Output.json() - model_data, message = PipelinedModel(role=role, party_id=party_id, job_id=job_id).read_model_data(task_name) - return get_json_result(code=ReturnCode.Base.SUCCESS, message=message, data=model_data) + +@manager.route('/data/download', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +@API.Input.params(output_key=fields.String(required=False), desc=OUTPUT_KEY) +def output_data_download(job_id, role, party_id, task_name, output_key=None): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return DataManager.download_output_data( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version, + output_key=output_key, + tar_file_name=f"{job_id}_{role}_{party_id}_{task_name}" + + ) + + +@manager.route('/data/table', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def output_data_table(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return DataManager.query_output_data_table( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version + ) + + +@manager.route('/data/display', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def output_data_display(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return DataManager.display_output_data( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version + ) diff --git a/python/fate_flow/apps/client/permission_app.py b/python/fate_flow/apps/client/permission_app.py new file mode 100644 index 000000000..fa461503a --- /dev/null +++ b/python/fate_flow/apps/client/permission_app.py @@ -0,0 +1,89 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import PERMISSION_APP_ID, PERMISSION_ROLE, PARTY_ID, COMPONENT, DATASET +from fate_flow.controller.app_controller import PermissionController +from fate_flow.controller.permission_controller import ResourcePermissionController +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.types import PermissionParameters +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import PERMISSION_MANAGER_PAGE +from fate_flow.utils.api_utils import API + +page_name = PERMISSION_MANAGER_PAGE + + +@manager.route('/grant', methods=['POST']) +@API.Input.json(app_id=fields.String(required=True), desc=PERMISSION_APP_ID) +@API.Input.json(role=fields.String(required=True), desc=PERMISSION_ROLE) +def grant(app_id, role): + for roles in PermissionController.get_roles_for_user(app_id=app_id): + PermissionController.delete_role_for_user(app_id=app_id, role=roles, grant_role=role) + status = PermissionController.add_role_for_user(app_id=app_id, role=role) + return API.Output.json(data={"status": status}) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(app_id=fields.String(required=True), desc=PERMISSION_APP_ID) +@API.Input.json(role=fields.String(required=True), desc=PERMISSION_ROLE) +def delete(app_id, role): + status = PermissionController.delete_role_for_user(app_id=app_id, role=role) + return API.Output.json(data={"status": status}) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(app_id=fields.String(required=True), desc=PERMISSION_APP_ID) +def query(app_id): + permissions = {} + for role in PermissionController.get_roles_for_user(app_id=app_id): + permissions[role] = PermissionController.get_permissions_for_user(app_id=role) + return API.Output.json(code=ReturnCode.Base.SUCCESS, data=permissions) + + +@manager.route('/role/query', methods=['GET']) +def query_roles(): + return API.Output.json(data=RuntimeConfig.CLIENT_ROLE) + + +@manager.route('/resource/grant', methods=['post']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(component=fields.String(required=False), desc=COMPONENT) +@API.Input.json(dataset=fields.List(fields.Dict(), required=False), desc=DATASET) +def grant_resource_permission(party_id, component=None, dataset=None): + parameters = PermissionParameters(party_id=party_id, component=component, dataset=dataset) + ResourcePermissionController(party_id).grant_or_delete(parameters) + return API.Output.json() + + +@manager.route('/resource/delete', methods=['post']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(component=fields.String(required=False), desc=COMPONENT) +@API.Input.json(dataset=fields.List(fields.Dict(), required=False), desc=DATASET) +def delete_resource_permission(party_id, component=None, dataset=None): + parameters = PermissionParameters(party_id=party_id, component=component, dataset=dataset, is_delete=True) + ResourcePermissionController(parameters.party_id).grant_or_delete(parameters) + return API.Output.json() + + +@manager.route('/resource/query', methods=['get']) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(component=fields.String(required=False), desc=COMPONENT) +@API.Input.params(dataset=fields.Dict(required=False), desc=DATASET) +def query_resource_privilege(party_id, component=None, dataset=None): + parameters = PermissionParameters(party_id=party_id, component=component, dataset=dataset) + data = ResourcePermissionController(parameters.party_id).query() + return API.Output.json(data=data) diff --git a/python/fate_flow/apps/client/provider_app.py b/python/fate_flow/apps/client/provider_app.py new file mode 100644 index 000000000..82ff5d845 --- /dev/null +++ b/python/fate_flow/apps/client/provider_app.py @@ -0,0 +1,55 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import PROVIDER_NAME, DEVICE, VERSION, COMPONENT_METADATA, PROVIDER_ALL_NAME +from fate_flow.errors.server_error import DeviceNotSupported +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.utils.api_utils import API + + +@manager.route('/register', methods=['POST']) +@API.Input.json(name=fields.String(required=True), desc=PROVIDER_NAME) +@API.Input.json(device=fields.String(required=True), desc=DEVICE) +@API.Input.json(version=fields.String(required=True), desc=VERSION) +@API.Input.json(metadata=fields.Dict(required=True), desc=COMPONENT_METADATA) +def register(name, device, version, metadata): + provider = ProviderManager.get_provider(name=name, device=device, version=version, metadata=metadata, check=True) + if provider: + operator_type = ProviderManager.register_provider(provider) + return API.Output.json(message=f"{operator_type} success") + else: + return API.Output.fate_flow_exception(DeviceNotSupported(device=device)) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(name=fields.String(required=False), desc=PROVIDER_NAME) +@API.Input.params(device=fields.String(required=False), desc=DEVICE) +@API.Input.params(version=fields.String(required=False), desc=VERSION) +@API.Input.params(provider_name=fields.String(required=False), desc=PROVIDER_ALL_NAME) +def query(name=None, device=None, version=None, provider_name=None): + providers = ProviderManager.query_provider(name=name, device=device, version=version, provider_name=provider_name) + return API.Output.json(data=[provider.to_human_model_dict() for provider in providers]) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(name=fields.String(required=False), desc=PROVIDER_NAME) +@API.Input.json(device=fields.String(required=False), desc=DEVICE) +@API.Input.json(version=fields.String(required=False), desc=VERSION) +@API.Input.json(provider_name=fields.String(required=False), desc=PROVIDER_ALL_NAME) +def delete(name=None, device=None, version=None, provider_name=None): + result = ProviderManager.delete_provider(name=name, device=device, version=version, provider_name=provider_name) + return API.Output.json(data=result) diff --git a/python/fate_flow/apps/client/server_app.py b/python/fate_flow/apps/client/server_app.py new file mode 100644 index 000000000..232d288e9 --- /dev/null +++ b/python/fate_flow/apps/client/server_app.py @@ -0,0 +1,93 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import SERVER_NAME, HOST, PORT, PROTOCOL, SERVICE_NAME, URI, METHOD, PARAMS, DATA, HEADERS +from fate_flow.errors.server_error import NoFoundServer +from fate_flow.manager.service.service_manager import ServiceRegistry, ServerRegistry +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.utils.api_utils import API + + +@manager.route('/fateflow', methods=['GET']) +def fate_flow_server_info(): + datas = RuntimeConfig.SERVICE_DB.get_servers(to_dict=True) + return API.Output.json(data=datas) + + +@manager.route('/query/all', methods=['GET']) +def query_all(): + data = ServerRegistry.get_all() + return API.Output.json(data=data) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(server_name=fields.String(required=True), desc=SERVER_NAME) +def query_server(server_name): + server_list = ServerRegistry.query_server_info_from_db(server_name) + if not server_list: + return API.Output.fate_flow_exception(NoFoundServer(server_name=server_name)) + return API.Output.json(data=server_list[0].to_human_model_dict()) + + +@manager.route('/registry', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.json(host=fields.String(required=True), desc=HOST) +@API.Input.json(port=fields.Integer(required=True), desc=PORT) +@API.Input.json(protocol=fields.String(required=False), desc=PROTOCOL) +def register_server(server_name, host, port, protocol="http"): + server_info = ServerRegistry.register(server_name, host, port, protocol) + return API.Output.json(data=server_info) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +def delete_server(server_name): + status = ServerRegistry.delete_server_from_db(server_name) + return API.Output.json(message="success" if status else "failed") + + +@manager.route('/service/query', methods=['GET']) +@API.Input.params(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.params(service_name=fields.String(required=True), desc=SERVICE_NAME) +def query_service(server_name, service_name): + service_list = ServiceRegistry.load_service(server_name=server_name, service_name=service_name) + if not service_list: + return API.Output.fate_flow_exception(NoFoundServer(server_name=server_name)) + return API.Output.json(data=service_list[0].to_human_model_dict()) + + +@manager.route('/service/registry', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.json(service_name=fields.String(required=True), desc=SERVICE_NAME) +@API.Input.json(uri=fields.String(required=True), desc=URI) +@API.Input.json(method=fields.String(required=False), desc=METHOD) +@API.Input.json(params=fields.Dict(required=False), desc=PARAMS) +@API.Input.json(data=fields.Dict(required=False), desc=DATA) +@API.Input.json(headers=fields.Dict(required=False), desc=HEADERS) +@API.Input.json(protocol=fields.String(required=False), desc=PROTOCOL) +def registry_service(server_name, service_name, uri, method="POST", params=None, data=None, headers=None, protocol="http"): + ServiceRegistry.save_service_info(server_name=server_name, service_name=service_name, uri=uri, method=method, + params=params, data=data, headers=headers, protocol=protocol) + return API.Output.json() + + +@manager.route('/service/delete', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.json(service_name=fields.String(required=True), desc=SERVICE_NAME) +def delete_service(server_name, service_name): + status = ServiceRegistry.delete(server_name, service_name) + return API.Output.json(message="success" if status else "failed") diff --git a/python/fate_flow/apps/client/site_app.py b/python/fate_flow/apps/client/site_app.py index 814a9a3cd..9a83bfcb2 100644 --- a/python/fate_flow/apps/client/site_app.py +++ b/python/fate_flow/apps/client/site_app.py @@ -13,16 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from webargs import fields - -from fate_flow.entity.types import ReturnCode -from fate_flow.settings import PARTY_ID, IS_STANDALONE -from fate_flow.utils.api_utils import get_json_result +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import IsStandalone +from fate_flow.runtime.system_settings import PARTY_ID, IS_STANDALONE +from fate_flow.utils.api_utils import API @manager.route('/info/query', methods=['GET']) def query_site_info(): if not IS_STANDALONE: - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success", data={"party_id": PARTY_ID}) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data={"party_id": PARTY_ID}) else: - return get_json_result(code=ReturnCode.Site.IS_STANDALONE, message="site is standalone") + return API.Output.fate_flow_exception(IsStandalone()) diff --git a/python/fate_flow/apps/client/table_app.py b/python/fate_flow/apps/client/table_app.py new file mode 100644 index 000000000..35bb6c6db --- /dev/null +++ b/python/fate_flow/apps/client/table_app.py @@ -0,0 +1,64 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import NAMESPACE, NAME, DISPLAY, SERVER_FILE_PATH +from fate_flow.engine import storage +from fate_flow.engine.storage import Session, StorageEngine +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundTable +from fate_flow.manager.data.data_manager import DataManager +from fate_flow.utils.api_utils import API + +page_name = "table" + + +@manager.route('/query', methods=['GET']) +@API.Input.params(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.params(name=fields.String(required=True), desc=NAME) +@API.Input.params(display=fields.Bool(required=False), desc=DISPLAY) +def query_table(namespace, name, display=False): + data, display_data = DataManager.get_data_info(namespace, name) + if data: + if display: + data.update({"display": display_data}) + return API.Output.json(data=data) + else: + return API.Output.fate_flow_exception(NoFoundTable(name=name, namespace=namespace)) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=True), desc=NAME) +def delete_table(namespace, name): + if DataManager.delete_data(namespace, name): + return API.Output.json() + else: + return API.Output.fate_flow_exception(NoFoundTable(name=name, namespace=namespace)) + + +@manager.route('/bind/path', methods=['POST']) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=True), desc=NAME) +@API.Input.json(path=fields.String(required=True), desc=SERVER_FILE_PATH) +def bind_path(namespace, name, path): + address = storage.StorageTableMeta.create_address(storage_engine=StorageEngine.PATH, address_dict={"path": path}) + storage_meta = storage.StorageTableBase( + namespace=namespace, name=name, address=address, + engine=StorageEngine.PATH, options=None, partitions=None + ) + storage_meta.create_meta() + return API.Output.json() diff --git a/python/fate_flow/apps/desc.py b/python/fate_flow/apps/desc.py new file mode 100644 index 000000000..e746a9d6b --- /dev/null +++ b/python/fate_flow/apps/desc.py @@ -0,0 +1,99 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# job +DAG_SCHEMA = "Definition and configuration of jobs, including the configuration of multiple tasks" +USER_NAME = "Username provided by the upper-level system" +JOB_ID = "Job ID" +ROLE = "Role of the participant: guest/host/arbiter/local" +STATUS = "Status of the job or task" +LIMIT = "Limit of rows or entries" +PAGE = "Page number" +DESCRIPTION = "Description information" +PARTNER = "Participant information" +ORDER_BY = "Field name for sorting" +ORDER = "Sorting order: asc/desc" + +# task +TASK_NAME = "Task name" +TASK_ID = "Task ID" +TASK_VERSION = "Task version" +NODES = "Tags and customizable information for tasks" + +# data +SERVER_FILE_PATH = "File path on the server" +SERVER_DIR_PATH = "Directory path on the server" +HEAD = "Whether the first row of the file is the data's head" +PARTITIONS = "Number of data partitions" +META = "Metadata of the data" +EXTEND_SID = "Whether to automatically fill a column as data row ID" +NAMESPACE = "Namespace of the data table" +NAME = "Name of the data table" +SITE_NAME = "Site name" +DATA_WAREHOUSE = "Data output, content like: {name: xxx, namespace: xxx}" +DROP = "Whether to destroy data if it already exists" +DOWNLOAD_HEADER = "Whether to download the data's head as the first row" + +# output +FILTERS = "Filter conditions" +OUTPUT_KEY = "Primary key for output data or model of the task" + +# table +DISPLAY = "Whether to return preview data" + +# server +SERVER_NAME = "Server name" +SERVICE_NAME = "Service name" +HOST = "Host IP" +PORT = "Service port" +PROTOCOL = "Protocol: http/https" +URI = "Service path" +METHOD = "Request method: POST/GET, etc." +PARAMS = "Request header parameters" +DATA = "Request body parameters" +HEADERS = "Request headers" + +# provider +PROVIDER_NAME = "Component provider name" +DEVICE = "Component running mode" +VERSION = "Component version" +COMPONENT_METADATA = "Detailed information about component registration" +PROVIDER_ALL_NAME = "Registered algorithm full name, provider + ':' + version + '@' + running mode, e.g., fate:2.0.0@local" + +# permission +PERMISSION_APP_ID = "App ID" +PERMISSION_ROLE = "Permission name" +COMPONENT = "Component name" +DATASET = "List of datasets" + +# log +LOG_TYPE = "Log level or type" +INSTANCE_ID = "Instance ID of the FATE Flow service" +BEGIN = "Starting line number" +END = "Ending line number" + +# site +PARTY_ID = "Site ID" + +# model +MODEL_ID = "Model ID" +MODEL_VERSION = "Model version" + +# app +APP_NAME = "App name for the client" +APP_ID = "App ID for the client" +SITE_APP_ID = "App ID for the site" +SITE_APP_TOKEN = "App token for the site" diff --git a/python/fate_flow/apps/desc_zh.py b/python/fate_flow/apps/desc_zh.py new file mode 100644 index 000000000..56593fcc6 --- /dev/null +++ b/python/fate_flow/apps/desc_zh.py @@ -0,0 +1,99 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# job +DAG_SCHEMA = "作业的定义和配置，包括多个任务的配置" +USER_NAME = "上层系统所提供的用户名" +JOB_ID = "作业ID" +ROLE = "参与方的角色: guest/host/arbiter/local" +STATUS = "作业或者任务的状态" +LIMIT = "限制条数或者行数" +PAGE = "页码数" +DESCRIPTION = "描述信息" +PARTNER = "参与方信息" +ORDER_BY = "排序的字段名" +ORDER = "排序方式：asc/desc" + +# task +TASK_NAME = "任务名称" +TASK_ID = "任务ID" +TASK_VERSION = "任务版本" +NODES = "任务的标签等信息，用户可自定义化" + +# data +SERVER_FILE_PATH = "服务器上的文件路径" +SERVER_DIR_PATH = "服务器上的目录路径" +HEAD = "文件的第一行是否为数据的Head" +PARTITIONS = "数据分区数量" +META = "数据的元信息" +EXTEND_SID = "是否需要自动填充一列作为数据行id" +NAMESPACE = "数据表的命名空间" +NAME = "数据表名" +DATA_WAREHOUSE = "数据输出，内容如:{name: xxx, namespace: xxx}" +DROP = "当数据存在时是否需要先销毁" +DOWNLOAD_HEADER = "是否需要下载数据的Head作为第一行" + +# output +FILTERS = "过滤条件" +OUTPUT_KEY = "任务的输出数据或者模型的主键" + +# table +DISPLAY = "是否需要返回预览数据" + +# server +SERVER_NAME = "服务器名称" +SERVICE_NAME = "服务名称" +HOST = "主机ip" +PORT = "服务端口" +PROTOCOL = "协议：http/https" +URI = "服务路径" +METHOD = "请求方式：POST/GET等" +PARAMS = "请求头参数" +DATA = "请求体参数" +HEADERS = "请求头" + +# provider +PROVIDER_NAME = "组件提供方名称" +DEVICE = "组件运行模式" +VERSION = "组件版本" +COMPONENT_METADATA = "组件注册详细信息" +PROVIDER_ALL_NAME = "注册的算法全名，提供方+':'+版本+'@'+运行模式，如: fate:2.0.0@local" + +# permission +PERMISSION_APP_ID = "App id" +PERMISSION_ROLE = "权限名称" +COMPONENT = "组件名" +DATASET = "数据集列表" + +# log +LOG_TYPE = "日志等级或类型" +INSTANCE_ID = "FATE Flow服务的实例ID" +BEGIN = "起始行号" +END = "结尾行号" + + +# site +PARTY_ID = "站点ID" + +# model +MODEL_ID = "模型id" +MODEL_VERSION = "模型版本" + +# app +APP_NAME = "客户端的app名称" +APP_ID = "客户端的app-id" +SITE_APP_ID = "站点的app-id" +SITE_APP_TOKEN = "站点的app-token" diff --git a/python/fate_flow/apps/partner/partner_app.py b/python/fate_flow/apps/partner/partner_app.py index 261521ad1..f12e9dbad 100644 --- a/python/fate_flow/apps/partner/partner_app.py +++ b/python/fate_flow/apps/partner/partner_app.py @@ -17,34 +17,48 @@ from fate_flow.controller.job_controller import JobController from fate_flow.controller.task_controller import TaskController -from fate_flow.entity.run_status import TaskStatus -from fate_flow.entity.types import ReturnCode -from fate_flow.manager.resource_manager import ResourceManager +from fate_flow.entity.types import TaskStatus +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import CreateJobFailed, UpdateJobFailed, KillFailed, JobResourceException,\ + NoFoundTask, StartTaskFailed, UpdateTaskFailed, KillTaskFailed, TaskResourceException +from fate_flow.manager.service.resource_manager import ResourceManager from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.api_utils import get_json_result, job_request_json, task_request_json +from fate_flow.utils.api_utils import API, stat_logger +from fate_flow.utils.wraps_utils import task_request_proxy, create_job_request_check page_name = 'partner' @manager.route('/job/create', methods=['POST']) -@job_request_json(dag_schema=fields.Dict(required=True)) +@API.Input.json(dag_schema=fields.Dict(required=True)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@create_job_request_check def partner_create_job(dag_schema, job_id, role, party_id): try: JobController.create_job(dag_schema, job_id, role, party_id) - return get_json_result(code=ReturnCode.Base.SUCCESS, message="create job success") - except RuntimeError as e: - return get_json_result(code=ReturnCode.Job.CREATE_JOB_FAILED, message=str(e), data={"job_id": job_id}) + return API.Output.json() + except Exception as e: + stat_logger.exception(e) + return API.Output.fate_flow_exception(CreateJobFailed(detail=str(e))) @manager.route('/job/start', methods=['POST']) -@job_request_json(extra_info=fields.Dict(required=False)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(extra_info=fields.Dict(required=False)) def start_job(job_id, role, party_id, extra_info=None): JobController.start_job(job_id=job_id, role=role, party_id=party_id, extra_info=extra_info) - return get_json_result(code=ReturnCode.Base.SUCCESS, message="start job success") + return API.Output.json() @manager.route('/job/status/update', methods=['POST']) -@job_request_json(status=fields.String(required=True)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(status=fields.String(required=True)) def partner_job_status_update(job_id, role, party_id, status): job_info = { "job_id": job_id, @@ -53,14 +67,18 @@ def partner_job_status_update(job_id, role, party_id, status): "status": status } if JobController.update_job_status(job_info=job_info): - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success') + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') else: - return get_json_result(code=ReturnCode.Job.UPDATE_STATUS_FAILED, - message="update job status does not take effect") + return API.Output.fate_flow_exception(UpdateJobFailed( + job_id=job_id, role=role, party_id=party_id, status=status + )) @manager.route('/job/update', methods=['POST']) -@job_request_json(progress=fields.Float()) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(progress=fields.Float()) def partner_job_update(job_id, role, party_id, progress): job_info = { "job_id": job_id, @@ -70,94 +88,128 @@ def partner_job_update(job_id, role, party_id, progress): if progress: job_info.update({"progress": progress}) if JobController.update_job(job_info=job_info): - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success') + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') else: - return get_json_result(code=ReturnCode.Job.UPDATE_FAILED, message="update job does not take effect") - - -@manager.route('/job/pipeline/save', methods=['POST']) -@job_request_json() -def save_pipeline(job_id, role, party_id): - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success') + return API.Output.fate_flow_exception(UpdateJobFailed(**job_info)) @manager.route('/job/resource/apply', methods=['POST']) -@job_request_json() +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) def apply_resource(job_id, role, party_id): status = ResourceManager.apply_for_job_resource(job_id, role, party_id) if status: - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success') + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') else: - return get_json_result(code=ReturnCode.Job.APPLY_RESOURCE_FAILED, - message=f'apply for job {job_id} resource failed') + return API.Output.fate_flow_exception(JobResourceException( + job_id=job_id, role=role, party_id=party_id, + operation_type="apply" + )) @manager.route('/job/resource/return', methods=['POST']) -@job_request_json() +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) def return_resource(job_id, role, party_id): status = ResourceManager.return_job_resource(job_id=job_id, role=role, party_id=party_id) if status: - return get_json_result(ReturnCode.Base.SUCCESS, message='success') + return API.Output.json(ReturnCode.Base.SUCCESS, message='success') else: - return get_json_result(code=ReturnCode.Job.APPLY_RESOURCE_FAILED, - message=f'return for job {job_id} resource failed') + return API.Output.fate_flow_exception(JobResourceException( + job_id=job_id, role=role, party_id=party_id, + operation_type="return" + )) @manager.route('/job/stop', methods=['POST']) -@job_request_json() +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) def stop_job(job_id, role, party_id): kill_status, kill_details = JobController.stop_jobs(job_id=job_id, role=role, party_id=party_id) - return get_json_result(code=ReturnCode.Base.SUCCESS if kill_status else ReturnCode.Job.KILL_FAILED, - message='success' if kill_status else 'failed', - data=kill_details) + if kill_status: + return API.Output.json() + return API.Output.fate_flow_exception(KillFailed(detail=kill_details)) @manager.route('/task/resource/apply', methods=['POST']) -@task_request_json() +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) def apply_task_resource(job_id, role, party_id, task_id, task_version): status = ResourceManager.apply_for_task_resource(job_id=job_id, role=role, party_id=party_id, task_id=task_id, task_version=task_version) if status: - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success') + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') else: - return get_json_result(code=ReturnCode.Task.APPLY_RESOURCE_FAILED, - message=f'apply for task {job_id} resource failed') + return API.Output.fate_flow_exception(TaskResourceException( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version, operation_type="apply" + )) @manager.route('/task/resource/return', methods=['POST']) -@task_request_json() +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) def return_task_resource(job_id, role, party_id, task_id, task_version): status = ResourceManager.return_task_resource(job_id=job_id, role=role, party_id=party_id, task_id=task_id, task_version=task_version) if status: - return get_json_result(ReturnCode.Base.SUCCESS, message='success') + return API.Output.json(ReturnCode.Base.SUCCESS, message='success') else: - return get_json_result(code=ReturnCode.Task.APPLY_RESOURCE_FAILED, - message=f'return for task {job_id} resource failed') + return API.Output.fate_flow_exception(TaskResourceException( + job_id=job_id, role=role, party_id=party_id, task_id=task_id, + task_version=task_version, operation_type="return" + )) @manager.route('/task/start', methods=['POST']) -@task_request_json() +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@task_request_proxy(filter_local=True) def start_task(job_id, role, party_id, task_id, task_version): if TaskController.start_task(job_id, role, party_id, task_id, task_version): - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success') + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') else: - return get_json_result(code=ReturnCode.Task.START_FAILED, message='start task failed') + return API.Output.fate_flow_exception(StartTaskFailed( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version + )) @manager.route('/task/collect', methods=['POST']) -@task_request_json() +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) def collect_task(job_id, role, party_id, task_id, task_version): task_info = TaskController.collect_task(job_id=job_id, task_id=task_id, task_version=task_version, role=role, party_id=party_id) if task_info: - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success", data=task_info) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=task_info) else: - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task not found") + return API.Output.fate_flow_exception(NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version)) @manager.route('/task/status/update', methods=['POST']) -@task_request_json(status=fields.String(required=True)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(status=fields.String(required=True)) def task_status_update(job_id, role, party_id, task_id, task_version, status): task_info = {} task_info.update({ @@ -169,16 +221,22 @@ def task_status_update(job_id, role, party_id, task_id, task_version, status): "status": status }) if TaskController.update_task_status(task_info=task_info): - return get_json_result(code=ReturnCode.Base.SUCCESS, message='success') + return API.Output.json() else: - return get_json_result( - code=ReturnCode.Task.UPDATE_STATUS_FAILED, - message="update job status does not take effect" + return API.Output.fate_flow_exception(UpdateTaskFailed( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version, status=status) ) @manager.route('/task/stop', methods=['POST']) -@task_request_json(status=fields.String()) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(status=fields.String()) +@task_request_proxy() def stop_task(job_id, role, party_id, task_id, task_version, status=None): if not status: status = TaskStatus.FAILED @@ -186,18 +244,23 @@ def stop_task(job_id, role, party_id, task_id, task_version, status=None): kill_status = True for task in tasks: kill_status = kill_status & TaskController.stop_task(task=task, stop_status=status) - return get_json_result(code=ReturnCode.Base.SUCCESS if kill_status else ReturnCode.Task.KILL_FAILED, - message='success' if kill_status else 'failed') + if kill_status: + return API.Output.json() + else: + return API.Output.fate_flow_exception(KillTaskFailed(job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version)) @manager.route('/task/rerun', methods=['POST']) -@task_request_json(new_version=fields.Integer()) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(new_version=fields.Integer()) def rerun_task(job_id, role, party_id, task_id, task_version, new_version): tasks = JobSaver.query_task(job_id=job_id, task_id=task_id, role=role, party_id=party_id) if not tasks: - return get_json_result( - code=ReturnCode.Task.NOT_FOUND, - message="task not found" - ) + return API.Output.fate_flow_exception(NoFoundTask(job_id=job_id, role=role, party_id=party_id, task_id=task_id)) TaskController.create_new_version_task(task=tasks[0], new_version=new_version) - return get_json_result() + return API.Output.json() diff --git a/python/fate_flow/apps/scheduler/scheduler_app.py b/python/fate_flow/apps/scheduler/scheduler_app.py index b82fe537f..6e0cf614c 100644 --- a/python/fate_flow/apps/scheduler/scheduler_app.py +++ b/python/fate_flow/apps/scheduler/scheduler_app.py @@ -14,25 +14,30 @@ # limitations under the License. from webargs import fields -from fate_flow.entity.dag_structures import DAGSchema +from fate_flow.errors.server_error import UpdateTaskFailed from fate_flow.operation.job_saver import ScheduleJobSaver from fate_flow.scheduler.job_scheduler import DAGScheduler -from fate_flow.utils.api_utils import get_json_result, validate_request_json, task_request_json +from fate_flow.utils.api_utils import API page_name = 'scheduler' @manager.route('/job/create', methods=['POST']) -@validate_request_json(dag_schema=fields.Dict(required=True)) +@API.Input.json(dag_schema=fields.Dict(required=True)) def create_job(dag_schema): - submit_result = DAGScheduler.submit(DAGSchema(**dag_schema)) - return get_json_result(**submit_result) + submit_result = DAGScheduler.submit(dag_schema) + return API.Output.json(**submit_result) @manager.route('/task/report', methods=['POST']) -@task_request_json(status=fields.String(required=False)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(status=fields.String(required=False)) def report_task(job_id, role, party_id, task_id, task_version, status=None): - ScheduleJobSaver.update_task_status(task_info={ + status = ScheduleJobSaver.update_task_status(task_info={ "job_id": job_id, "role": role, "party_id": party_id, @@ -40,19 +45,25 @@ def report_task(job_id, role, party_id, task_id, task_version, status=None): "task_version": task_version, "status": status }) - return get_json_result(code=0, message='success') + if status: + return API.Output.json() + return API.Output.fate_flow_exception(UpdateTaskFailed( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version, status=status) + ) @manager.route('/job/stop', methods=['POST']) -@validate_request_json(job_id=fields.String(required=True), stop_status=fields.String(required=False)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(stop_status=fields.String(required=False)) def stop_job(job_id, stop_status=None): retcode, retmsg = DAGScheduler.stop_job(job_id=job_id, stop_status=stop_status) - return get_json_result(code=retcode, message=retmsg) + return API.Output.json(code=retcode, message=retmsg) @manager.route('/job/rerun', methods=['POST']) -@validate_request_json(job_id=fields.String(required=True)) +@API.Input.json(job_id=fields.String(required=True)) def rerun_job(job_id): - DAGScheduler.set_job_rerun(job_id=job_id, auto=False) - return get_json_result() + DAGScheduler.rerun_job(job_id=job_id, auto=False) + return API.Output.json() diff --git a/python/fate_flow/apps/worker/worker_app.py b/python/fate_flow/apps/worker/worker_app.py index 8b410191d..89a6c6282 100644 --- a/python/fate_flow/apps/worker/worker_app.py +++ b/python/fate_flow/apps/worker/worker_app.py @@ -12,97 +12,183 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json + from flask import request from webargs import fields from fate_flow.controller.task_controller import TaskController -from fate_flow.entity.types import ReturnCode -from fate_flow.manager.model_manager import PipelinedModel -from fate_flow.manager.output_manager import OutputDataTracking, OutputMetric +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundTask +from fate_flow.manager.data.data_manager import DataManager +from fate_flow.manager.model.model_manager import PipelinedModel +from fate_flow.manager.metric.metric_manager import OutputMetric +from fate_flow.manager.service.output_manager import OutputDataTracking from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.api_utils import get_json_result, validate_request_json, validate_request_params +from fate_flow.utils.api_utils import API page_name = 'worker' @manager.route('/task/status', methods=['POST']) -@validate_request_json(status=fields.String(required=True), execution_id=fields.String(required=True), - error=fields.String(required=False)) +@API.Input.json(execution_id=fields.String(required=True)) +@API.Input.json(status=fields.String(required=True)) +@API.Input.json(error=fields.String(required=False)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) def report_task_status(status, execution_id, error=None): - tasks = JobSaver.query_task(execution_id=execution_id) - if tasks: - task = tasks[0] - task_info = { - "party_status": status, - "job_id": task.f_job_id, - "role": task.f_role, - "party_id": task.f_party_id, - "task_id": task.f_task_id, - "task_version": task.f_task_version - } - TaskController.update_task_status(task_info=task_info) - if error: - task_info.update({"error_report": error}) - TaskController.update_task(task_info) - return get_json_result() - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task not found") - - -@manager.route('/task/status', methods=['GET']) -@validate_request_params(execution_id=fields.String(required=True)) -def query_task_status(execution_id): - tasks = JobSaver.query_task(execution_id=execution_id) - if tasks: - task_info = { - "status": tasks[0].f_status, - } - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success", data=task_info) - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task not found") - - -@manager.route('/task/output/tracking', methods=['POST']) -@validate_request_json(execution_id=fields.String(required=True), meta_data=fields.Dict(required=True), - type=fields.String(required=True), uri=fields.String(required=True), - output_key=fields.String(required=True)) -def log_output_artifacts(execution_id, meta_data, type, uri, output_key): - tasks = JobSaver.query_task(execution_id=execution_id) - if tasks: - task = tasks[0] + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + task_info = { + "party_status": status, + "job_id": task.f_job_id, + "role": task.f_role, + "party_id": task.f_party_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version + } + TaskController.update_task_status(task_info=task_info) + if error: + task_info.update({"error_report": error}) + TaskController.update_task(task_info) + return API.Output.json() + + +@manager.route('/model/save', methods=['POST']) +@API.Input.form(model_id=fields.String(required=True)) +@API.Input.form(model_version=fields.String(required=True)) +@API.Input.form(execution_id=fields.String(required=True)) +@API.Input.form(output_key=fields.String(required=True)) +@API.Input.form(type_name=fields.String(required=True)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def upload_model(model_id, model_version, execution_id, output_key, type_name): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + file = request.files['file'] + PipelinedModel.upload_model( + model_file=file, + job_id=task.f_job_id, + task_name=task.f_task_name, + role=task.f_role, + party_id=task.f_party_id, + output_key=output_key, + model_id=model_id, + model_version=model_version, + type_name=type_name + ) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success") + + +@manager.route('/model/download', methods=['GET']) +@API.Input.params(model_id=fields.String(required=True)) +@API.Input.params(model_version=fields.String(required=True)) +@API.Input.params(role=fields.String(required=True)) +@API.Input.params(party_id=fields.String(required=True)) +@API.Input.params(task_name=fields.String(required=True)) +@API.Input.params(output_key=fields.String(required=True)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def download_model(model_id, model_version, role, party_id, task_name, output_key): + return PipelinedModel.download_model( + model_id=model_id, + model_version=model_version, + role=role, + party_id=party_id, + task_name=task_name, + output_key=output_key + ) + + +@manager.route('/data/tracking/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=False)) +@API.Input.params(role=fields.String(required=False)) +@API.Input.params(party_id=fields.String(required=False)) +@API.Input.params(task_name=fields.String(required=False)) +@API.Input.params(output_key=fields.String(required=False)) +@API.Input.params(namespace=fields.String(required=False)) +@API.Input.params(name=fields.String(required=False)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def query_data_tracking(job_id=None, role=None, party_id=None, task_name=None, output_key=None, namespace=None, name=None): + tracking_list = [] + if not namespace and not name: data_info = { - "type": type, - "uri": uri, - "output_key": output_key, - "meta": meta_data, - "job_id": task.f_job_id, - "role": task.f_role, - "party_id": task.f_party_id, - "task_id": task.f_task_id, - "task_version": task.f_task_version, - "task_name": task.f_task_name + "job_id": job_id, + "role": role, + "party_id": party_id, + "task_name": task_name, + "output_key": output_key } - OutputDataTracking.create(data_info) - return get_json_result(code=ReturnCode.Base.SUCCESS, message="success") - return get_json_result(code=ReturnCode.Task.NOT_FOUND, message="task not found") + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + data_info.update({ + "task_id": tasks[0].f_task_id, + "task_version": tasks[0].f_task_version + }) + data_list = OutputDataTracking.query(**data_info) + if not data_list: + return API.Output.json(code=ReturnCode.Task.NO_FOUND_MODEL_OUTPUT, message="failed") + for data in data_list: + info, _ = DataManager.get_data_info(data.f_namespace, data.f_name) + tracking_list.append(info) + else: + info, _ = DataManager.get_data_info(namespace, name) + tracking_list.append(info) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=tracking_list) -@manager.route('/task/model////////', methods=['POST']) -def save_output_model(job_id, role, party_id, model_id, model_version, component, task_name, model_name): - file = request.files['file'] - PipelinedModel(job_id=job_id, model_id=model_id, model_version=model_version, role=role, party_id=party_id).save_output_model( - task_name, model_name, component, model_file=file) - return get_json_result() + +@manager.route('/data/tracking/save', methods=['POST']) +@API.Input.json(execution_id=fields.String(required=True)) +@API.Input.json(meta_data=fields.Dict(required=True)) +@API.Input.json(uri=fields.String(required=True)) +@API.Input.json(output_key=fields.String(required=True)) +@API.Input.json(namespace=fields.String(required=True)) +@API.Input.json(name=fields.String(required=True)) +@API.Input.json(overview=fields.Dict(required=True)) +@API.Input.json(partitions=fields.Int(required=False)) +@API.Input.json(source=fields.Dict(required=True)) +@API.Input.json(data_type=fields.String(required=True)) +@API.Input.json(index=fields.Int(required=True)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def save_data_tracking(execution_id, meta_data, uri, output_key, namespace, name, overview, source, data_type, index, + partitions=None): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + data_info = { + "uri": uri, + "output_key": output_key, + "job_id": task.f_job_id, + "role": task.f_role, + "party_id": task.f_party_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version, + "task_name": task.f_task_name, + "namespace": namespace, + "name": name, + "index": index + } + OutputDataTracking.create(data_info) + DataManager.create_data_table( + namespace=namespace, name=name, uri=uri, partitions=partitions, + data_meta=meta_data, source=source, data_type=data_type, + count=overview.get("count", None), part_of_data=overview.get("samples", []) + ) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success") -@manager.route('/task/model////////', methods=['GET']) -def get_output_model(job_id, role, party_id, model_id, model_version, component, task_name, model_name): - return PipelinedModel( - model_id=model_id, model_version=model_version, job_id=job_id, role=role, party_id=party_id - ).read_output_model(task_name, model_name) +@manager.route('/metric/save', methods=["POST"]) +@API.Input.json(execution_id=fields.String(required=True)) +@API.Input.json(data=fields.List(fields.Dict())) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def save_metric(execution_id, data): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + OutputMetric(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, task_name=task.f_task_name, + task_id=task.f_task_id, task_version=task.f_task_version).save_output_metrics(data) + return API.Output.json() -@manager.route('/task/metric///////', methods=["POST"]) -@validate_request_json(data=fields.Dict(required=True), incomplete=fields.Bool(required=True)) -def save_metric(job_id, role, party_id, task_name, task_id, task_version, name, data, incomplete): - OutputMetric(job_id=job_id, role=role, party_id=party_id, task_name=task_name, task_id=task_id, - task_version=task_version).save_output_metrics(data, incomplete) - return get_json_result() +@manager.route('/metric/save/', methods=["POST"]) +@API.Input.json(data=fields.List(fields.Dict())) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def save_metrics(execution_id, data): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + OutputMetric(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, task_name=task.f_task_name, + task_id=task.f_task_id, task_version=task.f_task_version).save_output_metrics(data) + return API.Output.json() diff --git a/python/fate_flow/commands/__init__.py b/python/fate_flow/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/commands/server_cli.py b/python/fate_flow/commands/server_cli.py new file mode 100644 index 000000000..b0147e98a --- /dev/null +++ b/python/fate_flow/commands/server_cli.py @@ -0,0 +1,200 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import subprocess +import platform +import click +from ruamel import yaml + +import fate_flow +from fate_flow.commands.service import manage_fate_service + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +HOME = os.path.dirname(fate_flow.__file__) +SERVER_CONF_PATH = os.path.join(HOME, "conf", "service_conf.yaml") +SETTING_PATH = os.path.join(HOME, "settings.py") +SERVICE_SH = os.path.join(HOME, "commands", "service.sh") + + +@click.group(short_help='Fate Flow', context_settings=CONTEXT_SETTINGS) +@click.pass_context +def flow_server_cli(ctx): + ''' + Fate Flow server cli + ''' + ctx.ensure_object(dict) + if ctx.invoked_subcommand == 'init': + return + pass + + +@flow_server_cli.command('init', short_help='Flow Server init command') +@click.option('--ip', type=click.STRING, help='Fate flow server ip address.') +@click.option('--port', type=click.INT, help='Fate flow server http port.') +@click.option('--home', type=click.STRING, help="Service's home directory, used to store essential information " + "such as data, logs, and more.") +def initialization(**kwargs): + """ + \b + - DESCRIPTION: + Flow Init Command. provide ip and port of a valid fate flow server. + + \b + - USAGE: + fate_flow init --ip 127.0.0.1 --port 9380 --home /data/projects/fate_flow + + """ + init_server(kwargs.get("ip"), kwargs.get("port"), kwargs.get("home")) + + +@flow_server_cli.command('start', short_help='Start run flow server') +def start(**kwargs): + """ + \b + - DESCRIPTION: + Start FATE Flow Server Command. + + \b + - USAGE: + fate_flow start + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "start") + else: + run_command("start") + + +@flow_server_cli.command('status', short_help='Query fate flow server status') +def status(**kwargs): + """ + \b + - DESCRIPTION: + Query fate flow server status command + + \b + - USAGE: + fate_flow status + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "status") + else: + run_command("status") + + +@flow_server_cli.command('stop', short_help='Stop run flow server') +def stop(**kwargs): + """ + \b + - DESCRIPTION: + Stop FATE Flow Server Command. + + \b + - USAGE: + fate_flow stop + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "stop") + else: + run_command("stop") + + +@flow_server_cli.command('restart', short_help='Restart fate flow server') +def restart(**kwargs): + """ + \b + - DESCRIPTION: + ReStart FATE Flow Server Command. + + \b + - USAGE: + fate_flow restart + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "restart") + else: + run_command("restart") + + +@flow_server_cli.command('version', short_help='Flow Server Version Command') +def get_version(): + import fate_flow + print(fate_flow.__version__) + + +def replace_settings(home_path): + import re + with open(SETTING_PATH, "r") as file: + content = file.read() + content = re.sub(r"DATA_DIR.*", f"DATA_DIR = \"{home_path}/data\"", content) + content = re.sub(r"MODEL_DIR.*", f"MODEL_DIR = \"{home_path}/model\"", content) + content = re.sub(r"JOB_DIR.*", f"JOB_DIR = \"{home_path}/jobs\"", content) + content = re.sub(r"LOG_DIR.*", f"LOG_DIR = \"{home_path}/logs\"", content) + content = re.sub(r"SQLITE_FILE_NAME.*", f"SQLITE_FILE_NAME = \"{home_path}/fate_flow_sqlite.db\"", content) + with open(SETTING_PATH, "w") as file: + file.write(content) + + with open(SERVICE_SH, "r") as file: + content = file.read() + content = re.sub(r"LOG_DIR.*=.*", f"LOG_DIR=\"{home_path}/logs\"", content) + with open(SERVICE_SH, "w") as file: + file.write(content) + + +def init_server(ip, port, home): + with open(SERVER_CONF_PATH, "r") as file: + config = yaml.safe_load(file) + if ip: + print(f"ip: {ip}") + config["fateflow"]["host"] = ip + if port: + print(f"port: {port}") + config["fateflow"]["http_port"] = port + if home: + if not os.path.isabs(home): + raise RuntimeError(f"Please use an absolute path: {home}") + os.makedirs(home, exist_ok=True) + print(f"home: {home}") + replace_settings(home) + + if ip or port: + with open(SERVER_CONF_PATH, "w") as file: + yaml.dump(config, file) + + print("Init server completed!") + + +def run_command(command): + try: + command = f"sh {SERVICE_SH} {HOME} {command}" + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, text=True) + if result.returncode == 0: + print(result.stdout) + return result.stdout + else: + print(result.stdout) + print(f"Error: {result.stderr}") + return None + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + return command + + +if __name__ == '__main__': + flow_server_cli() diff --git a/python/fate_flow/commands/service.py b/python/fate_flow/commands/service.py new file mode 100644 index 000000000..6ea6cf3db --- /dev/null +++ b/python/fate_flow/commands/service.py @@ -0,0 +1,170 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import subprocess +import time +from ruamel import yaml + + +def load_yaml_conf(conf_path): + with open(conf_path) as f: + return yaml.safe_load(f) + + +def make_logs_dir(log_dir): + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + +def manage_fate_service(project_base, action): + parser = argparse.ArgumentParser(description='FATE Service Manager') + parser.add_argument('project_base', type=str, help='path to the FATE project directory') + parser.add_argument('action', choices=['start', 'stop', 'status', 'restart'], help='action to perform') + + args = parser.parse_args([project_base, action]) + print(f'project_base:{args.project_base},action:{args.action}') + http_port, grpc_port = get_ports(args.project_base) + if args.action == 'start': + start_service(args.project_base) + get_service_status(http_port, grpc_port) + elif args.action == 'stop': + stop_service(http_port, grpc_port) + elif args.action == 'status': + get_service_status(http_port, grpc_port) + elif args.action == 'restart': + stop_service(http_port, grpc_port) + time.sleep(2) + start_service(args.project_base) + get_service_status(http_port, grpc_port) + + +def get_ports(project_base): + service_conf_path = os.path.join(project_base, 'conf/service_conf.yaml') + if not os.path.isfile(service_conf_path): + print(f'service conf not found: {service_conf_path}') + exit(1) + + config = load_yaml_conf(service_conf_path) + http_port = config.get('fateflow').get('http_port') + grpc_port = config.get('fateflow').get('grpc_port') + print(f'fate flow http port: {http_port}, grpc port: {grpc_port}\n') + return http_port, grpc_port + + +def get_pid(http_port, grpc_port): + netstat_command = ["netstat", "-ano"] + output = subprocess.run(netstat_command, capture_output=True, text=True).stdout + + pid = None + lines = output.split('\n') + for line in lines: + parts = line.split() + if len(parts) >= 5: + protocol = parts[0] + local_address = parts[1] + state = parts[3] + if state == 'LISTENING' and ':' in local_address: + port = local_address.split(':')[-1] + _pid = parts[-1] + if port == str(http_port) or port == str(grpc_port): + pid = _pid + break + return pid + + +def get_service_status(http_port, grpc_port): + pid = get_pid(http_port, grpc_port) + if pid: + task_list = subprocess.getoutput(f"tasklist /FI \"PID eq {pid}\"") + print(f"status: {task_list}") + + print(f'LISTENING on port {http_port}:') + print(subprocess.getoutput(f'netstat -ano | findstr :{http_port}')) + + print(f'LISTENING on port {grpc_port}:') + print(subprocess.getoutput(f'netstat -ano | findstr :{grpc_port}')) + else: + print('service not running') + + +def start_service(project_base): + http_port = None + grpc_port = None + + service_conf_path = os.path.join(project_base, 'conf/service_conf.yaml') + if os.path.isfile(service_conf_path): + config = load_yaml_conf(service_conf_path) + http_port = config.get('fateflow').get('http_port') + grpc_port = config.get('fateflow').get('grpc_port') + + if not http_port or not grpc_port: + print(f'service conf not found or missing port information: {service_conf_path}') + exit(1) + + pid = get_pid(http_port, grpc_port) + if pid: + print(f'service already started. pid: {pid}') + return + + log_dir = os.path.join(project_base, 'logs') + make_logs_dir(log_dir) + + command = ['python', os.path.join(project_base, 'fate_flow_server.py')] + # print(f'command:{command}') + stdout = open(os.path.join(log_dir, 'console.log'), 'a') + stderr = open(os.path.join(log_dir, 'error.log'), 'a') + + subprocess.Popen(command, stdout=stdout, stderr=stderr) + + for _ in range(100): + time.sleep(0.1) + pid = get_pid(http_port, grpc_port) + if pid: + print(f'service started successfully. pid: {pid}') + return + + pid = get_pid(http_port, grpc_port) + if pid: + print(f'service started successfully. pid: {pid}') + else: + print( + f'service start failed, please check {os.path.join(log_dir, "error.log")} and {os.path.join(log_dir, "console.log")}') + + +def stop_service(http_port, grpc_port): + pid = get_pid(http_port, grpc_port) + if not pid: + print('service not running') + return + task_list = subprocess.getoutput(f"tasklist /FI \"PID eq {pid}\"") + print(f'killing: {task_list}') + + try: + subprocess.run(['taskkill', '/F', '/PID', str(pid)]) + time.sleep(1) + except subprocess.CalledProcessError: + print('failed to kill the process') + return + + if get_pid(http_port, grpc_port): + print('failed to stop the service') + else: + print('service stopped successfully') + + + + diff --git a/python/fate_flow/commands/service.sh b/python/fate_flow/commands/service.sh new file mode 100644 index 000000000..6e82dc27c --- /dev/null +++ b/python/fate_flow/commands/service.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +PROJECT_BASE=$1 +LOG_DIR=$PROJECT_BASE/logs + + +parse_yaml() { + local prefix=$2 + local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') + sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | + awk -F$fs '{ + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i> "${LOG_DIR}/console.log" 2>>"${LOG_DIR}/error.log" + else + nohup python $PROJECT_BASE/fate_flow_server.py >> "${LOG_DIR}/console.log" 2>>"${LOG_DIR}/error.log" & + fi + for((i=1;i<=100;i++)); + do + sleep 0.1 + getpid + if [[ -n ${pid} ]]; then + echo "service start sucessfully. pid: ${pid}" + return + fi + done + if [[ -n ${pid} ]]; then + echo "service start sucessfully. pid: ${pid}" + else + echo "service start failed, please check ${LOG_DIR}/error.log and ${LOG_DIR}/console.log" + fi + else + echo "service already started. pid: ${pid}" + fi +} + +stop() { + getpid + if [[ -n ${pid} ]]; then + echo "killing: `ps aux | grep ${pid} | grep -v grep`" + for((i=1;i<=100;i++)); + do + sleep 0.1 + kill ${pid} + getpid + if [[ ! -n ${pid} ]]; then + echo "killed by SIGTERM" + return + fi + done + kill -9 ${pid} + if [[ $? -eq 0 ]]; then + echo "killed by SIGKILL" + else + echo "kill error" + fi + else + echo "service not running" + fi +} + + +case "$2" in + start) + start + status + ;; + + starting) + start front + ;; + + stop) + stop + ;; + + status) + status + ;; + + restart) + stop + sleep 2 + start + status + ;; + *) + echo "usage: $0 {start|stop|status|restart}" + exit -1 +esac diff --git a/python/fate_flow/components/__init__.py b/python/fate_flow/components/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/components/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/components/components/__init__.py b/python/fate_flow/components/components/__init__.py new file mode 100644 index 000000000..64524f308 --- /dev/null +++ b/python/fate_flow/components/components/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .upload import upload +from .download import download + +BUILDIN_COMPONENTS = [upload, download] diff --git a/python/fate_flow/components/components/download.py b/python/fate_flow/components/components/download.py new file mode 100644 index 000000000..4be33737f --- /dev/null +++ b/python/fate_flow/components/components/download.py @@ -0,0 +1,68 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from fate_flow.components import cpn +from fate_flow.engine import storage +from fate_flow.errors.server_error import NoFoundTable +from fate_flow.manager.data.data_manager import DataManager + + +@cpn.component() +def download( + config +): + download_data(config) + + +def download_data(config): + job_id = config.pop("job_id") + download_object = Download() + download_object.run( + parameters=DownloadParam( + **config + ) + ) + + +class DownloadParam(object): + def __init__( + self, + namespace, + name, + path, + ): + self.name = name + self.namespace = namespace + self.path = path + + +class Download: + def __init__(self): + self.parameters = None + self.table = None + self.data_meta = {} + + def run(self, parameters: DownloadParam): + data_table_meta = storage.StorageTableMeta(name=parameters.name, namespace=parameters.namespace) + if not data_table_meta: + raise NoFoundTable(name=parameters.name, namespace=parameters.namespace) + download_dir = parameters.path + logging.info("start download data") + DataManager.send_table( + output_tables_meta={"data": data_table_meta}, + download_dir=download_dir + ) + logging.info(f"download data success, download path: {parameters.path}") diff --git a/python/fate_flow/components/components/upload.py b/python/fate_flow/components/components/upload.py new file mode 100644 index 000000000..0e8898a0c --- /dev/null +++ b/python/fate_flow/components/components/upload.py @@ -0,0 +1,327 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +import secrets +from typing import Union + +from fate_flow.components import cpn +from fate_flow.engine.storage import Session, StorageEngine, DataType, StorageTableMeta, StorageOrigin +from fate_flow.entity.spec.dag import ArtifactSource +from fate_flow.manager.data.data_manager import DatasetManager +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME +from fate_flow.utils.file_utils import get_fate_flow_directory + + +@cpn.component() +def upload( + config +): + upload_data(config) + + +def upload_data(config): + job_id = config.pop("job_id") + upload_object = Upload() + data = upload_object.run( + parameters=UploadParam( + **config + ), + job_id=job_id + ) + + +class Param(object): + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + if v is None: + continue + d[k] = v + return d + + +class MetaParam(Param): + def __init__( + self, + sample_id_name: str = None, + match_id_name: str = None, + match_id_list: list = None, + match_id_range: int = 0, + label_name: Union[None, str] = None, + label_type: str = "int32", + weight_name: Union[None, str] = None, + weight_type: str = "float32", + header: str = None, + delimiter: str = ",", + dtype: Union[str, dict] = "float32", + na_values: Union[str, list, dict] = None, + input_format: str = "dense", + tag_with_value: bool = False, + tag_value_delimiter: str = ":" + ): + self.sample_id_name = sample_id_name + self.match_id_name = match_id_name + self.match_id_list = match_id_list + self.match_id_range = match_id_range + self.label_name = label_name + self.label_type = label_type + self.weight_name = weight_name + self.weight_type = weight_type + self.header = header + self.delimiter = delimiter + self.dtype = dtype + self.na_values = na_values + self.input_format = input_format + self.tag_with_value = tag_with_value + self.tag_value_delimiter = tag_value_delimiter + + +class UploadParam(Param): + def __init__( + self, + namespace="", + name="", + file="", + storage_engine="", + head=1, + partitions=10, + extend_sid=False, + address: dict = {}, + meta: dict = {} + ): + self.name = name + self.namespace = namespace + self.file = file + self.storage_engine = storage_engine + self.head = head + self.partitions = partitions + self.extend_sid = extend_sid + self.meta = MetaParam(**meta) + self.storage_address = address + + +class Upload: + def __init__(self): + self.parameters = None + self.table = None + self.data_meta = {} + + def run(self, parameters: UploadParam, job_id=""): + self.parameters = parameters + logging.info(self.parameters.to_dict()) + storage_address = self.parameters.storage_address + if not os.path.isabs(parameters.file): + parameters.file = os.path.join( + get_fate_flow_directory(), parameters.file + ) + name, namespace = parameters.name, parameters.namespace + with Session() as sess: + # clean table + table = sess.get_table(namespace=namespace, name=name) + if table: + logging.info( + f"destroy table name: {name} namespace: {namespace} engine: {table.engine}" + ) + try: + table.destroy() + except Exception as e: + logging.error(e) + else: + logging.info( + f"can not found table name: {name} namespace: {namespace}, pass destroy" + ) + address_dict = storage_address.copy() + storage_engine = self.parameters.storage_engine + storage_session = sess.storage( + storage_engine=storage_engine + ) + if storage_engine in {StorageEngine.EGGROLL, StorageEngine.STANDALONE}: + upload_address = { + "name": name, + "namespace": namespace + } + if storage_engine == StorageEngine.STANDALONE: + upload_address.update({"home": STANDALONE_DATA_HOME}) + elif storage_engine in {StorageEngine.HDFS, StorageEngine.FILE}: + upload_address = { + "path": DatasetManager.upload_data_path( + name=name, + namespace=namespace, + storage_engine=storage_engine + ) + } + else: + raise RuntimeError(f"can not support this storage engine: {storage_engine}") + address_dict.update(upload_address) + logging.info(f"upload to {storage_engine} storage, address: {address_dict}") + address = StorageTableMeta.create_address( + storage_engine=storage_engine, address_dict=address_dict + ) + self.table = storage_session.create_table( + address=address, + source=ArtifactSource( + task_id="", + party_task_id="", + task_name="upload", + component="upload", + output_artifact_key="data" + ).dict(), + **self.parameters.to_dict() + ) + data_table_count = self.save_data_table(job_id) + logging.info("------------load data finish!-----------------") + + logging.info("file: {}".format(self.parameters.file)) + logging.info("total data_count: {}".format(data_table_count)) + logging.info("table name: {}, table namespace: {}".format(name, namespace)) + return {"name": name, "namespace": namespace, "count": data_table_count} + + def save_data_table(self, job_id): + input_file = self.parameters.file + input_feature_count = self.get_count(input_file) + self.upload_file(input_file, job_id, input_feature_count) + table_count = self.table.count() + metas_info = { + "count": table_count, + "partitions": self.parameters.partitions, + "data_type": DataType.TABLE + } + self.table.meta.update_metas(**metas_info) + return table_count + + def update_schema(self, fp): + id_index = 0 + read_status = False + if self.parameters.head is True: + data_head = fp.readline() + id_index = self.update_table_meta(data_head) + read_status = True + else: + pass + return id_index, read_status + + def upload_file(self, input_file, job_id, input_feature_count=None, table=None): + if not table: + table = self.table + part_of_data = [] + with open(input_file, "r") as fp: + id_index, read_status = self.update_schema(fp) + if read_status: + input_feature_count -= 1 + self.table.put_all(self.kv_generator(input_feature_count, fp, job_id, part_of_data, id_index=id_index)) + table.meta.update_metas(part_of_data=part_of_data) + + def get_line(self): + if not self.parameters.extend_sid: + line = self.get_data_line + else: + line = self.get_sid_data_line + return line + + @staticmethod + def get_data_line(values, delimiter, id_index, **kwargs): + if id_index: + k = values[id_index] + v = delimiter.join([ + delimiter.join(values[:id_index]), + delimiter.join(values[id_index + 1:]) + ]).strip(delimiter) + else: + k = values[0] + v = delimiter.join(list(map(str, values[1:]))) + return k, v + + @staticmethod + def get_sid_data_line(values, delimiter, fate_uuid, line_index, **kwargs): + return fate_uuid + str(line_index), delimiter.join(list(map(str, values[:]))) + + def kv_generator(self, input_feature_count, fp, job_id, part_of_data, id_index): + fate_uuid = secrets.token_bytes(16).hex() + get_line = self.get_line() + line_index = 0 + logging.info(input_feature_count) + while True: + lines = fp.readlines(104857600) + if lines: + for line in lines: + values = line.rstrip().split(self.parameters.meta.delimiter) + k, v = get_line( + values=values, + line_index=line_index, + delimiter=self.parameters.meta.delimiter, + fate_uuid=fate_uuid, + id_index=id_index + ) + yield k, v + line_index += 1 + if line_index <= 100: + part_of_data.append((k, v)) + save_progress = line_index / input_feature_count * 100 // 1 + job_info = { + "progress": save_progress, + "job_id": job_id, + "role": "local", + "party_id": 0, + } + logging.info(f"job info: {job_info}") + else: + return + + def get_count(self, input_file): + with open(input_file, "r", encoding="utf-8") as fp: + count = 0 + for line in fp: + count += 1 + return count + + def update_table_meta(self, data_head): + logging.info(f"data head: {data_head}") + update_schema, id_index = self.get_header_schema( + header_line=data_head + ) + self.data_meta.update(self.parameters.meta.to_dict()) + self.data_meta.update(update_schema) + self.table.meta.update_metas(data_meta=self.data_meta) + return id_index + + def get_header_schema(self, header_line): + delimiter = self.parameters.meta.delimiter + sample_id_name = self.parameters.meta.sample_id_name + sample_id_index = 0 + if self.parameters.extend_sid: + sample_id_name = "extend_sid" + header = delimiter.join([sample_id_name, header_line]).strip() + else: + header_list = header_line.split(delimiter) + if not sample_id_name: + # default set sample_id_index = 0 + sample_id_name = header_list[0] + else: + if sample_id_name not in header_line: + raise RuntimeError(f"No found sample id {sample_id_name} in header") + sample_id_index = header_list.index(sample_id_name) + if sample_id_index > 0: + header_line = self.join_in_index_line(delimiter, header_list, sample_id_index) + header = header_line.strip() + return {'header': header, "sample_id_name": sample_id_name}, sample_id_index + + @staticmethod + def join_in_index_line(delimiter, values, id_index): + return delimiter.join([ + values[id_index], + delimiter.join(values[:id_index]), + delimiter.join(values[id_index + 1:]) + ]).strip(delimiter) diff --git a/python/fate_flow/components/cpn.py b/python/fate_flow/components/cpn.py new file mode 100644 index 000000000..43fb828ca --- /dev/null +++ b/python/fate_flow/components/cpn.py @@ -0,0 +1,55 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import logging +from typing import Any + +from pydantic import BaseModel + + +class Params(BaseModel): + class TaskParams(BaseModel): + job_id: str + + component_params: Any + task_params: TaskParams + + +class _Component: + def __init__( + self, + name: str, + callback + ) -> None: + self.name = name + self.callback = callback + + def execute(self, config): + return self.callback(config) + + +def component(*args, **kwargs): + def decorator(f): + cpn_name = f.__name__.lower() + if isinstance(f, _Component): + raise TypeError("Attempted to convert a callback into a component twice.") + cpn = _Component( + name=cpn_name, + callback=f + ) + cpn.__doc__ = f.__doc__ + # cpn.validate_declare() + return cpn + return decorator diff --git a/python/fate_flow/components/entrypoint/__init__.py b/python/fate_flow/components/entrypoint/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/components/entrypoint/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/components/entrypoint/component.py b/python/fate_flow/components/entrypoint/component.py new file mode 100644 index 000000000..ae2240dff --- /dev/null +++ b/python/fate_flow/components/entrypoint/component.py @@ -0,0 +1,36 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from fate_flow.entity.spec.dag import TaskConfigSpec + +logger = logging.getLogger(__name__) + + +def execute_component(config: TaskConfigSpec): + component = load_component(config.component) + cpn_config = config.parameters + cpn_config["job_id"] = config.job_id + logger.info(f"cpn_config： {cpn_config}") + + component.execute(cpn_config) + + +def load_component(cpn_name: str): + from fate_flow.components.components import BUILDIN_COMPONENTS + + for cpn in BUILDIN_COMPONENTS: + if cpn.name == cpn_name: + return cpn diff --git a/python/fate_flow/controller/app_controller.py b/python/fate_flow/controller/app_controller.py new file mode 100644 index 000000000..f693869bc --- /dev/null +++ b/python/fate_flow/controller/app_controller.py @@ -0,0 +1,133 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import hashlib +import time + +from fate_flow.db.casbin_models import FATE_CASBIN +from fate_flow.errors.server_error import RequestExpired, NoFoundAppid, InvalidParameter, RoleTypeError +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import CLIENT_AUTHENTICATION, SITE_AUTHENTICATION +from fate_flow.utils.base_utils import generate_random_id +from fate_flow.utils.wraps_utils import switch_function, check_permission + + +class Authentication(object): + @classmethod + def md5_sign(cls, app_id, app_token, user_name, initiator_party_id, timestamp, nonce): + key = hashlib.md5(str(app_id + user_name + initiator_party_id + nonce + timestamp).encode("utf8")).hexdigest().lower() + sign = hashlib.md5(str(key + app_token).encode("utf8")).hexdigest().lower() + return sign + + @classmethod + def md5_verify(cls, app_id, timestamp, nonce, signature, user_name="", initiator_party_id=""): + if cls.check_if_expired(timestamp): + raise RequestExpired() + apps = AppManager.query_app(app_id=app_id) + if apps: + _signature = cls.md5_sign( + app_id=app_id, + app_token=apps[0].f_app_token, + user_name=user_name, + initiator_party_id=initiator_party_id, + timestamp=timestamp, + nonce=nonce + ) + return _signature == signature + else: + raise NoFoundAppid(app_id=app_id) + + @staticmethod + def generate_timestamp(): + return str(int(time.time()*1000)) + + @staticmethod + def generate_nonce(): + return generate_random_id(length=4, only_number=True) + + @staticmethod + def check_if_expired(timestamp, timeout=60): + expiration = int(timestamp) + timeout * 1000 + if expiration < int(time.time() * 1000): + return True + else: + return False + + +class PermissionController(object): + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def add_policy(role, resource, permission): + return FATE_CASBIN.add_policy(role, resource, permission) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + @check_permission(operate="grant", types="permission") + @AppManager.check_app_type + def add_role_for_user(app_id, role, init=False): + PermissionController.check_permission_role(role) + return FATE_CASBIN.add_role_for_user(app_id, role) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + @check_permission(operate="delete", types="permission") + # @AppManager.check_app_type + def delete_role_for_user(app_id, role, grant_role=None, init=False): + role_type = role + PermissionController.check_permission_role(role) + app_info = AppManager.query_app(app_id=app_id) + if grant_role == "super_client": + grant_role = "client" + if grant_role and grant_role != app_info[0].f_app_type: + raise RoleTypeError(role=grant_role) + return FATE_CASBIN.delete_role_for_suer(app_id, role_type) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + @check_permission(operate="query", types="permission") + def get_roles_for_user(app_id): + return FATE_CASBIN.get_roles_for_user(app_id) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def get_permissions_for_user(app_id): + return FATE_CASBIN.get_permissions_for_user(app_id) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + def delete_roles_for_user(app_id): + return FATE_CASBIN.delete_roles_for_user(app_id) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + def has_role_for_user(app_id, role): + return FATE_CASBIN.has_role_for_user(app_id, role) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + def enforcer(app_id, resource, permission): + return FATE_CASBIN.enforcer(app_id, resource, permission) + + @staticmethod + def check_permission_role(role): + if role not in RuntimeConfig.CLIENT_ROLE: + raise InvalidParameter(role=role) diff --git a/python/fate_flow/controller/config_manager.py b/python/fate_flow/controller/config_manager.py index 02ade20fd..c5db9ae41 100644 --- a/python/fate_flow/controller/config_manager.py +++ b/python/fate_flow/controller/config_manager.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from fate_flow.manager.resource_manager import ResourceManager +from fate_flow.manager.service.resource_manager import ResourceManager +from fate_flow.manager.service.service_manager import ServerRegistry from fate_flow.runtime.job_default_config import JobDefaultConfig @@ -22,3 +23,4 @@ class ConfigManager: def load(cls): JobDefaultConfig.load() ResourceManager.initialize() + ServerRegistry.load() diff --git a/python/fate_flow/controller/job_controller.py b/python/fate_flow/controller/job_controller.py index 4a4e054a9..08db997e8 100644 --- a/python/fate_flow/controller/job_controller.py +++ b/python/fate_flow/controller/job_controller.py @@ -13,32 +13,44 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +import shutil + from fate_flow.controller.task_controller import TaskController -from fate_flow.entity.dag_structures import DAGSchema, JobConfSpec -from fate_flow.entity.run_status import EndStatus, JobStatus, TaskStatus -from fate_flow.entity.types import ReturnCode -from fate_flow.manager.resource_manager import ResourceManager +from fate_flow.db import Job +from fate_flow.engine.storage import Session +from fate_flow.entity.spec.dag import DAGSchema, JobConfSpec, InheritConfSpec +from fate_flow.entity.types import EndStatus, JobStatus, TaskStatus +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundJob, InheritanceFailed +from fate_flow.manager.metric.metric_manager import OutputMetric +from fate_flow.manager.model.model_manager import PipelinedModel +from fate_flow.manager.model.model_meta import ModelMeta +from fate_flow.manager.service.output_manager import OutputDataTracking +from fate_flow.manager.service.resource_manager import ResourceManager from fate_flow.operation.job_saver import JobSaver +from fate_flow.runtime.runtime_config import RuntimeConfig from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.settings import PARTY_ID from fate_flow.utils.base_utils import current_timestamp +from fate_flow.utils.job_utils import get_job_log_directory, save_job_dag from fate_flow.utils.log_utils import schedule_logger class JobController(object): @classmethod - def request_create_job(cls, dag_schema: dict): + def request_create_job(cls, dag_schema: dict, user_name: str = None, is_local=False): dag_schema = DAGSchema(**dag_schema) - if not dag_schema.dag.conf: - dag_schema.dag.conf = JobConfSpec() - dag_schema.dag.conf.initiator_party_id = PARTY_ID - if not dag_schema.dag.conf.scheduler_party_id: - dag_schema.dag.conf.scheduler_party_id = PARTY_ID + RuntimeConfig.SCHEDULER.check_job_parameters(dag_schema, is_local) response = FederatedScheduler.request_create_job( party_id=dag_schema.dag.conf.scheduler_party_id, + initiator_party_id=dag_schema.dag.conf.initiator_party_id, command_body={ - "dag_schema": dag_schema.dict() + "dag_schema": dag_schema.dict(exclude_defaults=True) }) + if user_name and response.get("code") == ReturnCode.Base.SUCCESS: + JobSaver.update_job_user(job_id=response.get("job_id"), user_name=user_name) + if response and isinstance(response, dict) and response.get("code") == ReturnCode.Base.SUCCESS: + save_job_dag(job_id=response.get("job_id"), dag=dag_schema.dict(exclude_defaults=True)) return response @classmethod @@ -46,7 +58,7 @@ def request_stop_job(cls, job_id): schedule_logger(job_id).info(f"stop job on this party") jobs = JobSaver.query_job(job_id=job_id) if not jobs: - return {"code": ReturnCode.Job.NOT_FOUND, "message": "job not found"} + raise NoFoundJob(job_id=job_id) status = JobStatus.CANCELED kill_status, kill_details = JobController.stop_jobs(job_id=job_id, stop_status=status) schedule_logger(job_id).info(f"stop job on this party status {kill_status}") @@ -82,8 +94,12 @@ def create_job(cls, dag_schema: dict, job_id: str, role: str, party_id: str): "model_id": dag_schema.dag.conf.model_id, "model_version": dag_schema.dag.conf.model_version } + party_parameters, task_run, task_cores = RuntimeConfig.SCHEDULER.adapt_party_parameters(dag_schema, role) + schedule_logger(job_id).info(f"party_job_parameters: {party_parameters}") + schedule_logger(job_id).info(f"role {role} party_id {party_id} task run: {task_run}, task cores {task_cores}") + job_info.update(party_parameters) JobSaver.create_job(job_info=job_info) - TaskController.create_tasks(job_id, role, party_id, dag_schema) + TaskController.create_tasks(job_id, role, party_id, dag_schema, task_run=task_run, task_cores=task_cores) @classmethod def start_job(cls, job_id, role, party_id, extra_info=None): @@ -98,9 +114,22 @@ def start_job(cls, job_id, role, party_id, extra_info=None): if extra_info: schedule_logger(job_id).info(f"extra info: {extra_info}") job_info.update(extra_info) - cls.update_job_status(job_info=job_info) - cls.update_job(job_info=job_info) - schedule_logger(job_id).info(f"start job on {role} {party_id} successfully") + try: + cls.inheritance_job(job_id, role, party_id) + except Exception as e: + schedule_logger(job_id).exception(e) + job_info.update({"status": JobStatus.FAILED}) + finally: + cls.update_job_status(job_info=job_info) + cls.update_job(job_info=job_info) + schedule_logger(job_id).info(f"start job on {role} {party_id} {job_info.get('status')}") + + @classmethod + def inheritance_job(cls, job_id, role, party_id): + job = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id)[0] + if job.f_inheritance: + schedule_logger(job_id).info(f"start inherit job {job_id}, inheritance: {job.f_inheritance}") + JobInheritance.load(job) @classmethod def update_job_status(cls, job_info): @@ -158,6 +187,67 @@ def query_job(cls, **kwargs): query_filters[k] = v return JobSaver.query_job(**query_filters) + @classmethod + def query_job_list(cls, limit, page, job_id, description, partner, party_id, role, status, order_by, order, + user_name): + # Provided to the job display page + offset = limit * (page - 1) + query = {'tag': ('!=', 'submit_failed')} + if job_id: + query["job_id"] = ('contains', job_id) + if description: + query["description"] = ('contains', description) + if party_id: + query["party_id"] = ('contains', party_id) + if partner: + query["partner"] = ('contains', partner) + if role: + query["role"] = ('in_', set(role)) + if status: + query["status"] = ('in_', set(status)) + by = [] + if order_by: + by.append(order_by) + if order: + by.append(order) + if not by: + by = ['create_time', 'desc'] + if user_name: + query["user_name"] = ("==", user_name) + jobs, count = JobSaver.list_job(limit, offset, query, by) + jobs = [job.to_human_model_dict() for job in jobs] + for job in jobs: + job['partners'] = set() + for _r in job['parties']: + job['partners'].update(_r.get("party_id")) + job['partners'].discard(job['party_id']) + job['partners'] = sorted(job['partners']) + return count, jobs + + @classmethod + def query_task_list(cls, limit, page, job_id, role, party_id, task_name, order_by, order): + offset = limit * (page - 1) + + query = {} + if job_id: + query["job_id"] = job_id + if role: + query["role"] = role + if party_id: + query["party_id"] = party_id + if task_name: + query["task_name"] = task_name + by = [] + if by: + by.append(order_by) + if order: + by.append(order) + if not by: + by = ['create_time', 'desc'] + + tasks, count = JobSaver.list_task(limit, offset, query, by) + return count, [task.to_human_model_dict() for task in tasks] + @classmethod def query_tasks(cls, **kwargs): query_filters = {} @@ -165,3 +255,231 @@ def query_tasks(cls, **kwargs): if v is not None: query_filters[k] = v return JobSaver.query_task(**query_filters) + + @classmethod + def clean_queue(cls): + # stop waiting job + jobs = JobSaver.query_job(status=JobStatus.WAITING) + clean_status = {} + for job in jobs: + status = FederatedScheduler.request_stop_job(party_id=job.f_scheduler_party_id,job_id=job.f_job_id, stop_status=JobStatus.CANCELED) + clean_status[job.f_job_id] = status + return clean_status + + @classmethod + def clean_job(cls, job_id): + jobs = JobSaver.query_job(job_id=job_id) + tasks = JobSaver.query_task(job_id=job_id) + if not jobs: + raise NoFoundJob(job_id=job_id) + FederatedScheduler.request_stop_job( + party_id=jobs[0].f_scheduler_party_id,job_id=jobs[0].f_job_id, stop_status=JobStatus.CANCELED + ) + for task in tasks: + # metric + try: + OutputMetric(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, task_version=task.f_task_version).delete_metrics() + schedule_logger(task.f_job_id).info(f'delete {task.f_job_id} {task.f_role} {task.f_party_id}' + f' {task.f_task_name} metric data success') + except Exception as e: + pass + + # data + try: + datas = OutputDataTracking.query( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version + ) + with Session() as sess: + for data in datas: + table = sess.get_table(name=data.f_name, namespace=data.f_namespace) + if table: + table.destroy() + except Exception as e: + pass + + # model + try: + PipelinedModel.delete_model(job_id=task.f_job_id, role=task.f_role, + party_id=task.f_party_id, task_name=task.f_task_name) + schedule_logger(task.f_job_id).info(f'delete {task.f_job_id} {task.f_role} {task.f_party_id}' + f' {task.f_task_name} model success') + except Exception as e: + pass + # JobSaver.delete_job(job_id=job_id) + + @staticmethod + def add_notes(job_id, role, party_id, notes): + job_info = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "description": notes + } + return JobSaver.update_job(job_info) + + +class JobInheritance: + @classmethod + def check(cls, inheritance: InheritConfSpec = None): + if not inheritance: + return + if not inheritance.task_list: + raise InheritanceFailed( + task_list=inheritance.task_list, + position="dag_schema.dag.conf.inheritance.task_list" + ) + inheritance_jobs = JobSaver.query_job(job_id=inheritance.job_id) + inheritance_tasks = JobSaver.query_task(job_id=inheritance.job_id) + if not inheritance_jobs: + raise InheritanceFailed(job_id=inheritance.job_id, detail=f"no found job {inheritance.job_id}") + task_status = {} + for task in inheritance_tasks: + task_status[task.f_task_name] = task.f_status + + for task_name in inheritance.task_list: + if task_name not in task_status.keys(): + raise InheritanceFailed(job_id=inheritance.job_id, task_name=task_name, detail="no found task name") + elif task_status[task_name] not in [TaskStatus.SUCCESS, TaskStatus.PASS]: + raise InheritanceFailed( + job_id=inheritance.job_id, + task_name=task_name, + task_status=task_status[task_name], + detail=f"task status need in [{TaskStatus.SUCCESS}, {TaskStatus.PASS}]" + ) + # todo: parsing and judging whether job can be inherited + + @classmethod + def load(cls, job: Job): + # load inheritance: data、model、metric、logs + inheritance = InheritConfSpec(**job.f_inheritance) + source_task_list = JobSaver.query_task(job_id=inheritance.job_id, role=job.f_role, party_id=job.f_party_id) + task_list = JobSaver.query_task(job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id) + target_task_list = [task for task in task_list if task.f_task_name in inheritance.task_list] + cls.load_logs(job, inheritance) + cls.load_output_tracking(job.f_job_id, source_task_list, target_task_list) + cls.load_model_meta(job.f_job_id, source_task_list, target_task_list, job.f_model_id, job.f_model_version) + cls.load_metric(job.f_job_id, source_task_list, target_task_list) + cls.load_status(job.f_job_id, source_task_list, target_task_list) + + @classmethod + def load_logs(cls, job: Job, inheritance: InheritConfSpec): + schedule_logger(job.f_job_id).info("start load job logs") + for task_name in inheritance.task_list: + source_path = os.path.join(get_job_log_directory(inheritance.job_id), job.f_role, job.f_party_id, task_name) + target_path = os.path.join(get_job_log_directory(job.f_job_id), job.f_role, job.f_party_id, task_name) + if os.path.exists(source_path): + if os.path.exists(target_path): + shutil.rmtree(target_path) + shutil.copytree(source_path, target_path) + schedule_logger(job.f_job_id).info("load job logs success") + + @classmethod + def load_output_tracking(cls, job_id, source_task_list, target_task_list): + def callback(target_task, source_task): + output_tracking = OutputDataTracking.query( + job_id=source_task.f_job_id, + role=source_task.f_role, + party_id=source_task.f_party_id, + task_name=source_task.f_task_name, + task_id=source_task.f_task_id, + task_version=source_task.f_task_version + ) + for t in output_tracking: + _t = t.to_human_model_dict() + _t.update({ + "job_id": target_task.f_job_id, + "task_id": target_task.f_task_id, + "task_version": target_task.f_task_version, + "role": target_task.f_role, + "party_id": target_task.f_party_id + }) + OutputDataTracking.create(_t) + schedule_logger(job_id).info("start load output tracking") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load output tracking success") + + @classmethod + def load_model_meta(cls, job_id, source_task_list, target_task_list, model_id, model_version): + def callback(target_task, source_task): + _model_metas = ModelMeta.query( + job_id=source_task.f_job_id, + role=source_task.f_role, + party_id=source_task.f_party_id, + task_name=source_task.f_task_name + ) + for _meta in _model_metas: + _md = _meta.to_human_model_dict() + _md.update({ + "job_id": target_task.f_job_id, + "task_id": target_task.f_task_id, + "task_version": target_task.f_task_version, + "role": target_task.f_role, + "party_id": target_task.f_party_id, + "model_id": model_id, + "model_version": model_version + }) + ModelMeta.save(**_md) + schedule_logger(job_id).info("start load model meta") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load model meta success") + + @classmethod + def load_metric(cls, job_id, source_task_list, target_task_list): + def callback(target_task, source_task): + OutputMetric( + job_id=source_task.f_job_id, + role=source_task.f_role, + party_id=source_task.f_party_id, + task_name=source_task.f_task_name, + task_id=source_task.f_task_id, + task_version=source_task.f_task_version + ).save_as( + job_id=target_task.f_job_id, + role=target_task.f_role, + party_id=target_task.f_party_id, + task_name=target_task.f_task_name, + task_id=target_task.f_task_id, + task_version=target_task.f_task_version + ) + schedule_logger(job_id).info("start load metric") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load metric success") + + @classmethod + def load_status(cls, job_id, source_task_list, target_task_list): + def callback(target_task, source_task): + task_info = { + "job_id": target_task.f_job_id, + "task_id": target_task.f_task_id, + "task_version": target_task.f_task_version, + "role": target_task.f_role, + "party_id": target_task.f_party_id + } + update_info = {} + update_list = ["cmd", "elapsed", "end_time", "engine_conf", "party_status", "run_ip", + "run_pid", "start_time", "status", "worker_id"] + for k in update_list: + update_info[k] = getattr(source_task, f"f_{k}") + task_info.update(update_info) + schedule_logger(task_info["job_id"]).info( + "try to update task {} {}".format(task_info["task_id"], task_info["task_version"])) + schedule_logger(task_info["job_id"]).info("update info: {}".format(update_info)) + JobSaver.update_task(task_info) + TaskController.update_task_status(task_info) + schedule_logger(job_id).info("start load status") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load status success") + + @staticmethod + def load_do(source_task_list, target_task_list, callback): + for source_task in source_task_list: + for target_task in target_task_list: + if target_task.f_task_name == source_task.f_task_name: + callback(target_task, source_task) diff --git a/python/fate_flow/controller/permission_controller.py b/python/fate_flow/controller/permission_controller.py new file mode 100644 index 000000000..d55b48a49 --- /dev/null +++ b/python/fate_flow/controller/permission_controller.py @@ -0,0 +1,124 @@ +from fate_flow.db.casbin_models import PERMISSION_CASBIN as PC +from fate_flow.errors.server_error import NoPermission, PermissionOperateError +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.utils.log_utils import getLogger +from fate_flow.entity.types import PermissionParameters, DataSet, PermissionType +from fate_flow.hook.common.parameters import PermissionReturn + + +logger = getLogger("permission") + + +class ResourcePermissionController: + def __init__(self, party_id): + self.party_id = party_id + self.casbin_controller = PC + if not self.casbin_controller: + raise PermissionOperateError(message="No permission controller is found") + + def check(self, permission_type, value): + logger.info(f"check source party id {self.party_id} {permission_type} {value}") + result = self.casbin_controller.enforce(self.party_id, permission_type, value) + logger.info(f"result: {result}") + return result + + def grant_or_delete(self, permission_parameters: PermissionParameters): + logger.info(f"{'grant' if not permission_parameters.is_delete else 'delete'} parameters:" + f" {permission_parameters.to_dict()}") + self.check_parameters(permission_parameters) + for permission_type in PermissionType.values(): + permission_value = getattr(permission_parameters, permission_type) + if permission_value: + if permission_value != "*": + if permission_type in [PermissionType.COMPONENT.value]: + value_list = [value.strip() for value in permission_value.split(self.value_delimiter)] + elif permission_type in [PermissionType.DATASET.value]: + if isinstance(permission_value, list): + value_list = [DataSet(**value).casbin_value for value in permission_value] + else: + value_list = [DataSet(**permission_value).casbin_value] + else: + raise PermissionOperateError(type=permission_type, message="Not Supported") + for value in value_list: + if not permission_parameters.is_delete: + self.casbin_controller.grant(self.party_id, permission_type, value) + else: + self.casbin_controller.delete(self.party_id, permission_type, value) + else: + if not permission_parameters.is_delete: + for value in self.all_value(permission_type): + self.casbin_controller.grant(self.party_id, permission_type, value) + else: + self.casbin_controller.delete_all(self.party_id, permission_type) + + def query(self): + result = {PermissionType.DATASET.value: [], PermissionType.COMPONENT.value: []} + for casbin_result in self.casbin_controller.query(self.party_id): + if casbin_result[1] == PermissionType.DATASET.value: + casbin_result[2] = DataSet.load_casbin_value(casbin_result[2]) + result[casbin_result[1]].append(casbin_result[2]) + return result + + def check_parameters(self, permission_parameters): + for permission_type in PermissionType.values(): + permission_value = getattr(permission_parameters, permission_type) + if permission_value: + if permission_type in [PermissionType.COMPONENT.value]: + if permission_value != "*": + value_list = [value.strip() for value in permission_value.split(self.value_delimiter)] + self.check_values(permission_type, value_list) + if permission_type in [PermissionType.DATASET.value]: + if isinstance(permission_value, list): + for dataset in permission_value: + DataSet(**dataset).check() + elif isinstance(permission_value, dict): + DataSet(**permission_value).check() + elif permission_value == "*": + pass + else: + raise PermissionOperateError(type=permission_type, value=permission_value) + + def check_values(self, permission_type, values): + error_value = [] + value_list = self.all_value(permission_type) + for value in values: + if value not in value_list: + error_value.append(value) + if error_value: + raise PermissionOperateError(type=permission_type, value=error_value) + + def all_value(self, permission_type): + if permission_type == PermissionType.COMPONENT.value: + value_list = self.all_component + else: + raise PermissionOperateError(type=permission_type, message="Not Support Grant all") + return value_list + + @property + def all_component(self): + return ProviderManager.get_all_components() + + @property + def value_delimiter(self): + return "," + + +class PermissionCheck(object): + def __init__(self, party_id, component_list, dataset_list, **kwargs): + self.component_list = component_list + self.dataset_list = dataset_list + self.controller = ResourcePermissionController(party_id) + + def check_component(self) -> PermissionReturn: + for component_name in self.component_list: + if not self.controller.check(PermissionType.COMPONENT.value, component_name): + e = NoPermission(type=PermissionType.COMPONENT.value, component_name=component_name) + return PermissionReturn(code=e.code, message=e.message) + return PermissionReturn() + + def check_dataset(self) -> PermissionReturn: + for dataset in self.dataset_list: + if not self.controller.check(PermissionType.DATASET.value, dataset.casbin_value): + e = NoPermission(type=PermissionType.DATASET.value, dataset=dataset.value) + return PermissionReturn(e.code, e.message) + return PermissionReturn() diff --git a/python/fate_flow/controller/task_controller.py b/python/fate_flow/controller/task_controller.py index 830f96bb1..5f193f20f 100644 --- a/python/fate_flow/controller/task_controller.py +++ b/python/fate_flow/controller/task_controller.py @@ -13,17 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import copy import os +import yaml + from fate_flow.db.db_models import Task from fate_flow.db.schedule_models import ScheduleTask, ScheduleJob, ScheduleTaskStatus -from fate_flow.engine.computing import build_engine -from fate_flow.entity.dag_structures import DAGSchema +from fate_flow.engine.devices import build_engine +from fate_flow.entity.spec.dag import DAGSchema, LauncherSpec from fate_flow.hub.flow_hub import FlowHub -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.manager.worker_manager import WorkerManager +from fate_flow.manager.service.resource_manager import ResourceManager +from fate_flow.manager.service.worker_manager import WorkerManager +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.runtime.runtime_config import RuntimeConfig from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.entity.run_status import EndStatus, TaskStatus, FederatedSchedulingStatusCode +from fate_flow.entity.types import EndStatus, TaskStatus, FederatedCommunicationType, LauncherType +from fate_flow.entity.code import FederatedSchedulingStatusCode from fate_flow.operation.job_saver import JobSaver, ScheduleJobSaver from fate_flow.utils import job_utils from fate_flow.utils.base_utils import current_timestamp, json_dumps @@ -34,29 +40,29 @@ class TaskController(object): INITIATOR_COLLECT_FIELDS = ["status", "party_status", "start_time", "update_time", "end_time", "elapsed"] @classmethod - def create_tasks(cls, job_id: str, role: str, party_id: str, dag_schema: DAGSchema, is_scheduler=False): + def create_tasks(cls, job_id: str, role: str, party_id: str, dag_schema: DAGSchema, task_run=None, task_cores=None, + is_scheduler=False): schedule_logger(job_id).info(f"start create {'scheduler' if is_scheduler else 'partner'} tasks ...") job_parser = FlowHub.load_job_parser(dag_schema) task_list = job_parser.topological_sort() for task_name in task_list: - cls.create_task(job_id, role, party_id, task_name, dag_schema, job_parser, is_scheduler) + cls.create_task(job_id, role, party_id, task_name, dag_schema, job_parser, task_run=task_run, + is_scheduler=is_scheduler, task_cores=task_cores) schedule_logger(job_id).info("create tasks success") @classmethod - def create_task(cls, job_id, role, party_id, task_name, dag_schema, job_parser, is_scheduler, task_version=0): - + def create_task(cls, job_id, role, party_id, task_name, dag_schema, job_parser, is_scheduler, task_run=None, + task_cores=None, task_version=0): task_id = job_utils.generate_task_id(job_id=job_id, component_name=task_name) execution_id = job_utils.generate_session_id(task_id, task_version, role, party_id) task_node = job_parser.get_task_node(task_name=task_name) - task_parser = FlowHub.load_task_parser( + task_parser = job_parser.task_parser( task_node=task_node, job_id=job_id, task_name=task_name, role=role, party_id=party_id, - task_id=task_id, execution_id=execution_id, task_version=task_version, parties=dag_schema.dag.parties + task_id=task_id, execution_id=execution_id, task_version=task_version, parties=dag_schema.dag.parties, + model_id=dag_schema.dag.conf.model_id, model_version=dag_schema.dag.conf.model_version ) need_run = task_parser.need_run schedule_logger(job_id).info(f"task {task_name} role {role} part id {party_id} need run status {need_run}") - task_parameters = task_parser.task_parameters.dict() - schedule_logger(job_id).info(f"task {task_name} role {role} part id {party_id} task_parameters" - f" {task_parameters}") if is_scheduler: if need_run: task = ScheduleTask() @@ -71,6 +77,11 @@ def create_task(cls, job_id, role, party_id, task_name, dag_schema, job_parser, task.f_parties = [party.dict() for party in dag_schema.dag.parties] ScheduleJobSaver.create_task(task.to_human_model_dict()) else: + task_parameters = task_parser.task_parameters + task_parameters.engine_run = task_run + task_parameters.computing_partitions = dag_schema.dag.conf.computing_partitions + schedule_logger(job_id).info(f"task {task_name} role {role} part id {party_id} task_parameters" + f" {task_parameters.dict()}, provider: {task_parser.provider}") task = Task() task.f_job_id = job_id task.f_role = role @@ -82,10 +93,35 @@ def create_task(cls, job_id, role, party_id, task_name, dag_schema, job_parser, task.f_scheduler_party_id = dag_schema.dag.conf.scheduler_party_id task.f_status = TaskStatus.WAITING if need_run else TaskStatus.PASS task.f_party_status = TaskStatus.WAITING - task.f_component_parameters = task_parameters task.f_execution_id = execution_id + task.f_provider_name = task_parser.provider + task.f_sync_type = dag_schema.dag.conf.sync_type + task.f_task_run = task_run + task.f_task_cores = task_cores + cls.update_local(task) + cls.update_launcher_config(task, task_parser.task_runtime_launcher, task_parameters) + task.f_component_parameters = task_parameters.dict() JobSaver.create_task(task.to_human_model_dict()) + @staticmethod + def update_local(task): + # HA need route to local + if task.f_role == "local": + task.f_run_ip = RuntimeConfig.JOB_SERVER_HOST + task.f_run_port = RuntimeConfig.HTTP_PORT + + @staticmethod + def update_launcher_config(task, task_runtime_launcher, task_parameters): + # support deepspeed and other launcher + schedule_logger(task.f_job_id).info(f"task runtime launcher: {task_runtime_launcher}") + launcher = LauncherSpec.parse_obj(task_runtime_launcher) + if launcher.name and launcher.name != LauncherType.DEFAULT: + task_parameters.launcher_name = task.f_launcher_name = launcher.name + launcher_conf = copy.deepcopy(JobDefaultConfig.launcher.get(task_parameters.launcher_name)) + if launcher.conf: + launcher_conf.update(launcher.conf) + task_parameters.launcher_conf = task.f_launcher_conf = launcher_conf + @staticmethod def create_schedule_tasks(job: ScheduleJob, dag_schema): for party in job.f_parties: @@ -111,7 +147,7 @@ def create_scheduler_tasks_status(cls, job_id, dag_schema, task_version=0, auto_ "task_version": task_version, "status": TaskStatus.WAITING, "auto_retries": dag_schema.dag.conf.auto_retries if auto_retries is None else auto_retries, - "federated_status_collect_type": dag_schema.dag.conf.federated_status_collect_type + "sync_type": dag_schema.dag.conf.sync_type } ScheduleJobSaver.create_task_scheduler_status(task_info) schedule_logger(job_id).info("create schedule task status success") @@ -132,25 +168,17 @@ def start_task(cls, job_id, role, party_id, task_id, task_version): try: task = JobSaver.query_task(task_id=task_id, task_version=task_version, role=role, party_id=party_id)[0] run_parameters = task.f_component_parameters - # update runtime parameters - job = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id)[0] - dag_schema = DAGSchema(**job.f_dag) - job_parser = FlowHub.load_job_parser(dag_schema) - task_node = job_parser.get_task_node(task_name=task.f_task_name) - task_parser = FlowHub.load_task_parser( - task_node=task_node, job_id=job_id, task_name=task.f_task_name, role=role, - party_id=party_id, parties=dag_schema.dag.parties - ) - task_parser.update_runtime_artifacts(run_parameters) schedule_logger(job_id).info(f"task run parameters: {run_parameters}") task_executor_process_start_status = False - config_dir = job_utils.get_task_directory(job_id, role, party_id, task.f_task_name, task_id, task_version) + config_dir = job_utils.get_task_directory( + job_id, role, party_id, task.f_task_name, task.f_task_version, input=True + ) os.makedirs(config_dir, exist_ok=True) - run_parameters_path = os.path.join(config_dir, 'task_parameters.json') + run_parameters_path = os.path.join(config_dir, 'preprocess_parameters.yaml') with open(run_parameters_path, 'w') as fw: - fw.write(json_dumps(run_parameters, indent=True)) - backend_engine = build_engine() + yaml.dump(run_parameters, fw) + backend_engine = build_engine(task.f_provider_name) run_info = backend_engine.run(task=task, run_parameters=run_parameters, run_parameters_path=run_parameters_path, @@ -187,8 +215,8 @@ def create_new_version_task(cls, task: Task, new_version): dag_schema = DAGSchema(**jobs[0].f_dag) job_parser = FlowHub.load_job_parser(dag_schema) cls.create_task( - task.f_job_id, task.f_role, task.f_party_id, task.f_task_name, dag_schema, job_parser, is_scheduler=False, - task_version=new_version + task.f_job_id, task.f_role, task.f_party_id, task.f_task_name, dag_schema, job_parser, + task_run=task.f_task_run, task_cores=task.f_task_cores, is_scheduler=False, task_version=new_version ) @classmethod @@ -247,20 +275,16 @@ def update_task(cls, task_info): return update_status @classmethod - def update_task_status(cls, task_info, scheduler_party_id=None): - if not scheduler_party_id: - scheduler_party_id = JobSaver.query_task( + def update_task_status(cls, task_info, scheduler_party_id=None, sync_type=None): + if not scheduler_party_id or not sync_type: + task = JobSaver.query_task( task_id=task_info.get("task_id"), task_version=task_info.get("task_version") - )[0].f_scheduler_party_id + )[0] + scheduler_party_id, sync_type = task.f_scheduler_party_id, task.f_sync_type update_status = JobSaver.update_task_status(task_info=task_info) if update_status and EndStatus.contains(task_info.get("party_status")): ResourceManager.return_task_resource(**task_info) - cls.clean_task(job_id=task_info["job_id"], - task_id=task_info["task_id"], - task_version=task_info["task_version"], - role=task_info["role"], - party_id=task_info["party_id"]) if "party_status" in task_info: report_task_info = { "job_id": task_info.get("job_id"), @@ -270,7 +294,8 @@ def update_task_status(cls, task_info, scheduler_party_id=None): "task_version": task_info.get("task_version"), "status": task_info.get("party_status") } - cls.report_task_to_scheduler(task_info=report_task_info, scheduler_party_id=scheduler_party_id) + if sync_type == FederatedCommunicationType.CALLBACK: + cls.report_task_to_scheduler(task_info=report_task_info, scheduler_party_id=scheduler_party_id) return update_status @classmethod @@ -298,7 +323,7 @@ def stop_task(cls, task: Task, stop_status): "party_status": stop_status, "kill_status": True } - cls.update_task_status(task_info=task_info, scheduler_party_id=task.f_scheduler_party_id) + cls.update_task_status(task_info=task_info, scheduler_party_id=task.f_scheduler_party_id, sync_type=task.f_sync_type) cls.update_task(task_info=task_info) return kill_status @@ -306,9 +331,10 @@ def stop_task(cls, task: Task, stop_status): def kill_task(cls, task: Task): kill_status = False try: - backend_engine = build_engine() + backend_engine = build_engine(task.f_provider_name) if backend_engine: backend_engine.kill(task) + backend_engine.cleanup(task) WorkerManager.kill_task_all_workers(task) except Exception as e: schedule_logger(task.f_job_id).exception(e) @@ -325,5 +351,12 @@ def kill_task(cls, task: Task): return kill_status @classmethod - def clean_task(cls, job_id, task_id, task_version, role, party_id): - pass + def clean_task(cls, task): + try: + backend_engine = build_engine(task.f_provider_name) + if backend_engine: + schedule_logger(task.f_job_id).info(f"start clean task:[{task.f_task_id} {task.f_task_version}]") + backend_engine.cleanup(task) + WorkerManager.kill_task_all_workers(task) + except Exception as e: + schedule_logger(task.f_job_id).exception(e) diff --git a/python/fate_flow/db/__init__.py b/python/fate_flow/db/__init__.py index 17749217b..1556878ce 100644 --- a/python/fate_flow/db/__init__.py +++ b/python/fate_flow/db/__init__.py @@ -15,4 +15,4 @@ # from fate_flow.db.storage_models import * from fate_flow.db.schedule_models import * -from fate_flow.db.db_models import * \ No newline at end of file +from fate_flow.db.db_models import * diff --git a/python/fate_flow/db/base_models.py b/python/fate_flow/db/base_models.py index efa61ec64..9a9f23161 100644 --- a/python/fate_flow/db/base_models.py +++ b/python/fate_flow/db/base_models.py @@ -29,25 +29,19 @@ IntegerField, Metadata, Model, - TextField, - Insert + TextField ) from playhouse.pool import PooledMySQLDatabase -from fate_flow.runtime.runtime_config import RuntimeConfig -from fate_flow.settings import DATABASE, IS_STANDALONE, stat_logger, FORCE_USE_SQLITE +from fate_flow.hub.flow_hub import FlowHub + +from fate_flow.runtime.system_settings import DATABASE from fate_flow.utils.base_utils import json_dumps, json_loads, date_string_to_timestamp, \ current_timestamp, timestamp_to_date -from fate_flow.utils.file_utils import get_fate_flow_directory from fate_flow.utils.log_utils import getLogger, sql_logger from fate_flow.utils.object_utils import from_dict_hook -if IS_STANDALONE or FORCE_USE_SQLITE: - from playhouse.apsw_ext import DateTimeField -else: - from peewee import DateTimeField - -CONTINUOUS_FIELD_TYPE = {IntegerField, FloatField, DateTimeField} +CONTINUOUS_FIELD_TYPE = {IntegerField, FloatField} AUTO_DATE_TIMESTAMP_FIELD_PREFIX = { "create", "start", @@ -147,6 +141,7 @@ def is_continuous_field(cls: typing.Type) -> bool: else: return False + class JsonSerializedField(SerializedField): def __init__(self, object_hook=from_dict_hook, object_pairs_hook=None, **kwargs): super(JsonSerializedField, self).__init__(serialized_type=SerializedType.JSON, object_hook=object_hook, @@ -180,19 +175,10 @@ def remove_field_name_prefix(field_name): @singleton class BaseDataBase: def __init__(self): - database_config = DATABASE.copy() - db_name = database_config.pop("name") - if IS_STANDALONE or FORCE_USE_SQLITE: - # sqlite does not support other options - Insert.on_conflict = lambda self, *args, **kwargs: self.on_conflict_replace() - - from playhouse.apsw_ext import APSWDatabase - self.database_connection = APSWDatabase(get_fate_flow_directory("fate_sqlite.db")) - RuntimeConfig.init_config(USE_LOCAL_DATABASE=True) - stat_logger.info('init sqlite database on standalone mode successfully') - else: - self.database_connection = PooledMySQLDatabase(db_name, **database_config) - stat_logger.info('init mysql database on cluster mode successfully') + engine_name = DATABASE.get("engine") + config = DATABASE.get(engine_name) + decrypt_key = DATABASE.get("decrypt_key") + self.database_connection = FlowHub.load_database(engine_name, config, decrypt_key) class DatabaseLock: @@ -253,9 +239,7 @@ def close_connection(): class BaseModel(Model): f_create_time = BigIntegerField(null=True) - f_create_date = DateTimeField(null=True) f_update_time = BigIntegerField(null=True) - f_update_date = DateTimeField(null=True) def to_json(self): # This function is obsolete @@ -294,7 +278,7 @@ def getter_by(cls, attr): return operator.attrgetter(attr)(cls) @classmethod - def query(cls, reverse=None, order_by=None, **kwargs): + def query(cls, reverse=None, order_by=None, force=False, **kwargs): filters = [] for f_n, f_v in kwargs.items(): attr_name = "f_%s" % f_n @@ -332,20 +316,35 @@ def query(cls, reverse=None, order_by=None, **kwargs): if filters: query_records = cls.select().where(*filters) if reverse is not None: - if not order_by or not hasattr(cls, f"f_{order_by}"): - order_by = "create_time" - if reverse is True: - query_records = query_records.order_by( - cls.getter_by(f"f_{order_by}").desc() - ) - elif reverse is False: - query_records = query_records.order_by( - cls.getter_by(f"f_{order_by}").asc() - ) + if isinstance(order_by, str) or not order_by: + if not order_by or not hasattr(cls, f"f_{order_by}"): + order_by = "create_time" + query_records = cls.desc(query_records=query_records, reverse=[reverse], order_by=[order_by]) + elif isinstance(order_by, list): + if not isinstance(reverse, list) or len(reverse) != len(order_by): + raise ValueError(f"reverse need is list and length={len(order_by)}") + query_records = cls.desc(query_records=query_records, reverse=reverse, order_by=order_by) + else: + raise ValueError(f"order_by type {type(order_by)} not support") + return [query_record for query_record in query_records] + + elif force: + # force query all + query_records = cls.select() return [query_record for query_record in query_records] else: return [] + @classmethod + def desc(cls, query_records, order_by: list, reverse: list): + _filters = list() + for _k, _ob in enumerate(order_by): + if reverse[_k] is True: + _filters.append(cls.getter_by(f"f_{_ob}").desc()) + else: + _filters.append(cls.getter_by(f"f_{_ob}").asc()) + return query_records.order_by(*tuple(_filters)) + @classmethod def insert(cls, __data=None, **insert): if isinstance(__data, dict) and __data: @@ -411,7 +410,7 @@ def fill_db_model_object(model_object, human_model_dict): class BaseModelOperate: @classmethod @DB.connection_context() - def _create_entity(cls, entity_model: object, entity_info: object) -> object: + def _create_entity(cls, entity_model, entity_info: dict) -> object: obj = entity_model() obj.f_create_time = current_timestamp() for k, v in entity_info.items(): @@ -424,15 +423,38 @@ def _create_entity(cls, entity_model: object, entity_info: object) -> object: raise Exception("Create {} failed".format(entity_model)) return obj except peewee.IntegrityError as e: - if e.args[0] == 1062 or (isinstance(e.args[0], str) and "UNIQUE constraint failed" in e.args[0]): - sql_logger(job_id=entity_info.get("job_id", "fate_flow")).warning(e) - else: - raise Exception("Create {} failed:\n{}".format(entity_model, e)) + # if e.args[0] == 1062 or (isinstance(e.args[0], str) and "UNIQUE constraint failed" in e.args[0]): + # sql_logger(job_id=entity_info.get("job_id", "fate_flow")).warning(e) + # else: + # raise Exception("Create {} failed:\n{}".format(entity_model, e)) + pass except Exception as e: raise Exception("Create {} failed:\n{}".format(entity_model, e)) @classmethod @DB.connection_context() - def _query(cls, entity_model, **kwargs) -> object: - return entity_model.query(**kwargs) + def _query(cls, entity_model, force=False, **kwargs): + return entity_model.query(force=force, **kwargs) + + @classmethod + @DB.connection_context() + def _delete(cls, entity_model, **kwargs): + _kwargs = {} + filters = [] + for f_k, f_v in kwargs.items(): + attr_name = "f_%s" % f_k + filters.append(operator.attrgetter(attr_name)(entity_model) == f_v) + return entity_model.delete().where(*filters).execute() > 0 + + @classmethod + def safe_save(cls, model, defaults, **kwargs): + entity_model, status = model.get_or_create( + **kwargs, + defaults=defaults) + if status is False: + for key in defaults: + setattr(entity_model, key, defaults[key]) + entity_model.save(force_insert=False) + return "update" + return "create" diff --git a/python/fate_flow/db/casbin_models.py b/python/fate_flow/db/casbin_models.py new file mode 100644 index 000000000..ca8b23212 --- /dev/null +++ b/python/fate_flow/db/casbin_models.py @@ -0,0 +1,243 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from functools import reduce + +import casbin +import peewee as pw + +from fate_flow.db.base_models import singleton, DB +from fate_flow.runtime.system_settings import CASBIN_MODEL_CONF, CASBIN_TABLE_NAME, PERMISSION_TABLE_NAME, \ + PERMISSION_CASBIN_MODEL_CONF + + +class FlowCasbinAdapter(casbin.persist.Adapter): + def __init__(self, rule=None): + if not rule: + rule = FlowCasbinRule + self.rule = rule + self.database = DB + proxy = pw.Proxy() + self.rule._meta.database = proxy + proxy.initialize(DB) + + def load_policy(self, model): + for line in self.rule.select(): + casbin.persist.load_policy_line(str(line), model) + + def _save_policy_line(self, ptype, rule): + data = dict(zip(['v0', 'v1', 'v2', 'v3', 'v4', 'v5'], rule)) + item = self.rule(ptype=ptype) + item.__data__.update(data) + item.save() + + def save_policy(self, model): + """saves all policy rules to the storage.""" + for sec in ["p", "g"]: + if sec not in model.model.keys(): + continue + for ptype, ast in model.model[sec].items(): + for rule in ast.policy: + self._save_policy_line(ptype, rule) + return True + + def add_policy(self, sec, ptype, rule): + """adds a policy rule to the storage.""" + self._save_policy_line(ptype, rule) + + def remove_policy(self, sec, ptype, rule): + """removes a policy rule from the storage.""" + if sec in ["p", "g"]: + condition = [self.rule.ptype==ptype] + data = dict(zip(['v0', 'v1', 'v2', 'v3', 'v4', 'v5'], rule)) + condition.extend([getattr(self.rule, k) == data[k] for k in data]) + check = self.rule.select().filter(*condition) + if check.exists(): + self.rule.delete().where(*condition).execute() + return True + else: + return False + else: + return False + + def remove_filtered_policy(self, sec, ptype, field_index, *field_values): + """removes policy rules that match the filter from the storage. + This is part of the Auto-Save feature. + """ + pass + + +class FlowCasbinRule(pw.Model): + class Meta: + table_name = CASBIN_TABLE_NAME + ptype = pw.CharField(max_length=255, null=True) + v0 = pw.CharField(max_length=255, null=True) + v1 = pw.CharField(max_length=255, null=True) + v2 = pw.CharField(max_length=255, null=True) + v3 = pw.CharField(max_length=255, null=True) + v4 = pw.CharField(max_length=255, null=True) + v5 = pw.CharField(max_length=255, null=True) + + def __str__(self): + return reduce(lambda x, y: str(x) + ', ' + str(y) if y else x, + [self.ptype, self.v0, self.v1, self.v2, self.v3, self.v4, self.v5]) + + def __repr__(self): + if not self.id: + return "<{cls}: {desc}>".format(cls=self.__class__.__name__, desc=self) + return "<{cls} {pk}: {desc}>".format(cls=self.__class__.__name__, pk=self.id, desc=self) + + +class PermissionCasbinRule(pw.Model): + class Meta: + table_name = PERMISSION_TABLE_NAME + ptype = pw.CharField(max_length=255, null=True) + v0 = pw.CharField(max_length=255, null=True) + v1 = pw.CharField(max_length=255, null=True) + v2 = pw.CharField(max_length=255, null=True) + v3 = pw.CharField(max_length=255, null=True) + v4 = pw.CharField(max_length=255, null=True) + v5 = pw.CharField(max_length=255, null=True) + + def __str__(self): + return reduce(lambda x, y: str(x) + ', ' + str(y) if y else x, + [self.ptype, self.v0, self.v1, self.v2, self.v3, self.v4, self.v5]) + + def __repr__(self): + if not self.id: + return "<{cls}: {desc}>".format(cls=self.__class__.__name__, desc=self) + return "<{cls} {pk}: {desc}>".format(cls=self.__class__.__name__, pk=self.id, desc=self) + + +class FlowEnforcer(casbin.Enforcer): + @property + def reload_policy(self): + self.load_policy() + return self + + +@singleton +class FateCasbin(object): + def __init__(self): + self.adapter = None + self.init_adapter() + self._e = FlowEnforcer(CASBIN_MODEL_CONF, self.adapter) + + def init_adapter(self): + self.adapter = FlowCasbinAdapter() + self.init_table() + + @staticmethod + def init_table(): + FlowCasbinRule.create_table() + + @property + def re(self) -> casbin.Enforcer: + return self._e.reload_policy + + @property + def e(self) -> casbin.Enforcer: + return self._e + + def add_policy(self, role, resource, permission): + return self.e.add_policy(role, resource, permission) + + def remove_policy(self, role, resource, permission): + return self.e.remove_policy(role, resource, permission) + + def add_role_for_user(self, user, role): + return self.e.add_role_for_user(user, role) + + def delete_role_for_suer(self, user, role): + return self.e.delete_role_for_user(user, role) + + def delete_roles_for_user(self, user): + return self.e.delete_roles_for_user(user) + + def delete_user(self, user): + return self.e.delete_user(user) + + def delete_role(self, role): + return self.e.delete_role(role) + + def delete_permission(self, *permission): + return self.e.delete_permission(*permission) + + def delete_permissions_for_user(self, user): + return self.e.delete_permissions_for_user(user) + + def get_roles_for_user(self, user): + return self.re.get_roles_for_user(user) + + def get_users_for_role(self, role): + return self.re.get_users_for_role(role) + + def has_role_for_user(self, user, role): + return self.re.has_role_for_user(user, role) + + def has_permission_for_user(self, user, *permission): + return self.re.has_permission_for_user(user, *permission) + + def get_permissions_for_user(self, user): + return self.re.get_permissions_for_user(user) + + def enforcer(self, *rvals): + return self.re.enforce(*rvals) + + +@singleton +class PermissionCasbin(object): + def __init__(self): + self.adapter = None + self.init_adapter() + self._e = FlowEnforcer(PERMISSION_CASBIN_MODEL_CONF, self.adapter) + + def init_adapter(self): + self.adapter = FlowCasbinAdapter(rule=PermissionCasbinRule) + self.init_table() + + @staticmethod + def init_table(): + PermissionCasbinRule.create_table() + + @property + def re(self) -> casbin.Enforcer: + return self._e.reload_policy + + @property + def e(self) -> casbin.Enforcer: + return self._e + + def query(self, party_id): + return self.re.get_permissions_for_user(party_id) + + def delete(self, party_id, type, value): + return self.re.delete_permission_for_user(party_id, type, value) + + def delete_all(self, party_id, type): + return self.re.remove_filtered_policy(0, party_id, type) + + def grant(self, party_id, type, value): + return self.re.add_permission_for_user(party_id, type, value) + + def enforce(self, party_id, type, value): + try: + return self.re.enforce(party_id, type, str(value)) + except Exception as e: + raise Exception(f"{party_id}, {type}, {value} {e}") + + +FATE_CASBIN = FateCasbin() +PERMISSION_CASBIN = PermissionCasbin() diff --git a/python/fate_flow/db/db_models.py b/python/fate_flow/db/db_models.py index e6a3e3837..b3826d1dd 100644 --- a/python/fate_flow/db/db_models.py +++ b/python/fate_flow/db/db_models.py @@ -16,13 +16,12 @@ import datetime from peewee import CharField, TextField, BigIntegerField, IntegerField, BooleanField, CompositeKey, BigAutoField -from fate_flow.db.base_models import DataBaseModel, JSONField, DateTimeField +from fate_flow.db.base_models import DataBaseModel, JSONField class Job(DataBaseModel): - # multi-party common configuration f_job_id = CharField(max_length=25, index=True) - f_name = CharField(max_length=500, null=True, default='') + f_user_name = CharField(max_length=500, null=True, default='') f_description = TextField(null=True, default='') f_tag = CharField(max_length=50, null=True, default='') f_dag = JSONField() @@ -32,15 +31,17 @@ class Job(DataBaseModel): f_scheduler_party_id = CharField(max_length=50) f_status = CharField(max_length=50) f_status_code = IntegerField(null=True) + + f_inheritance = JSONField(null=True) + # this party configuration f_role = CharField(max_length=50, index=True) f_party_id = CharField(max_length=50, index=True) f_progress = IntegerField(null=True, default=0) f_model_id = CharField(max_length=100, null=True) - f_model_version = IntegerField(null=True, default=0) + f_model_version = CharField(max_length=10) f_engine_name = CharField(max_length=50, null=True) - f_engine_type = CharField(max_length=10, null=True) f_cores = IntegerField(default=0) f_memory = IntegerField(default=0) # MB f_remaining_cores = IntegerField(default=0) @@ -50,9 +51,7 @@ class Job(DataBaseModel): f_return_resource_time = BigIntegerField(null=True) f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) f_elapsed = BigIntegerField(null=True) class Meta: @@ -73,6 +72,10 @@ class Task(DataBaseModel): f_status = CharField(max_length=50, index=True) f_status_code = IntegerField(null=True) f_component_parameters = JSONField(null=True) + f_task_run = JSONField(null=True) + f_memory = IntegerField(default=0) + f_task_cores = IntegerField(default=0) + f_resource_in_use = BooleanField(default=False) f_worker_id = CharField(null=True, max_length=100) f_cmd = JSONField(null=True) @@ -80,16 +83,18 @@ class Task(DataBaseModel): f_run_port = IntegerField(null=True) f_run_pid = IntegerField(null=True) f_party_status = CharField(max_length=50) - f_provider_info = JSONField(null=True) + f_provider_name = CharField(max_length=50) f_task_parameters = JSONField(null=True) f_engine_conf = JSONField(null=True) f_kill_status = BooleanField(default=False) f_error_report = TextField(default="") + f_sync_type = CharField(max_length=20) + + f_launcher_name = CharField(max_length=20, null=True) + f_launcher_conf = JSONField(null=True) f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) f_elapsed = BigIntegerField(null=True) class Meta: @@ -105,40 +110,14 @@ class TrackingOutputInfo(DataBaseModel): f_role = CharField(max_length=50, index=True) f_party_id = CharField(max_length=50, index=True) f_output_key = CharField(max_length=30) - f_type = CharField(max_length=10, null=True) + f_index = IntegerField() f_uri = CharField(max_length=200, null=True) - f_meta = JSONField() + f_namespace = CharField(max_length=200) + f_name = CharField(max_length=200) class Meta: - db_table = "t_tracking_output" - primary_key = CompositeKey('f_job_id', 'f_task_id', 'f_task_version', 'f_role', 'f_party_id', 'f_type', 'f_output_key') - - -class PipelineModelInfo(DataBaseModel): - f_role = CharField(max_length=50) - f_party_id = CharField(max_length=50) - f_job_id = CharField(max_length=25, index=True) - f_model_id = CharField(max_length=100, index=True) - f_model_version = IntegerField(index=True) - - class Meta: - db_table = "t_model_info" - primary_key = CompositeKey('f_job_id') - - -class PipelineModelMeta(DataBaseModel): - f_model_id = CharField(max_length=100) - f_model_version = IntegerField() - f_job_id = CharField(max_length=100, index=True) - f_role = CharField(max_length=50, index=True) - f_party_id = CharField(max_length=50, index=True) - f_task_name = CharField(max_length=100, index=True) - f_component = CharField(max_length=30, null=True) - f_model_name = CharField(max_length=30, null=True) - - class Meta: - db_table = 't_model_meta' - primary_key = CompositeKey('f_job_id', 'f_role', 'f_party_id', 'f_task_name', 'f_model_name') + db_table = "t_tracking_data_output" + primary_key = CompositeKey('f_job_id', 'f_task_id', 'f_task_version', 'f_role', 'f_party_id', 'f_output_key', 'f_uri') class EngineRegistry(DataBaseModel): @@ -149,7 +128,6 @@ class EngineRegistry(DataBaseModel): f_memory = IntegerField() # MB f_remaining_cores = IntegerField() f_remaining_memory = IntegerField() # MB - f_nodes = IntegerField() class Meta: db_table = "t_engine_registry" @@ -171,9 +149,7 @@ class WorkerInfo(DataBaseModel): f_config = JSONField(null=True) f_cmd = JSONField(null=True) f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) class Meta: db_table = "t_worker" @@ -205,13 +181,78 @@ class Meta: f_job_id = CharField(max_length=25, index=True) f_role = CharField(max_length=10, index=True) f_party_id = CharField(max_length=50) - f_task_name = CharField(max_length=30, index=True) + f_task_name = CharField(max_length=50, index=True) f_task_id = CharField(max_length=100) f_task_version = BigIntegerField(null=True) - f_namespace = CharField(max_length=30, index=True, null=True) f_name = CharField(max_length=30, index=True) - f_type = CharField() + f_type = CharField(max_length=30, index=True, null=True) f_groups = JSONField() - f_metadata = JSONField() + f_step_axis = CharField(max_length=30, index=True, null=True) f_data = JSONField() - f_incomplete = BooleanField() + + +class ProviderInfo(DataBaseModel): + f_provider_name = CharField(max_length=100, primary_key=True) + f_name = CharField(max_length=20, index=True) + f_version = CharField(max_length=20) + f_device = CharField(max_length=20) + f_metadata = JSONField() + + class Meta: + db_table = "t_provider_info" + + +class ComponentInfo(DataBaseModel): + f_provider_name = CharField(max_length=100) + f_name = CharField(max_length=20, index=True) + f_version = CharField(max_length=20) + f_device = CharField(max_length=20) + f_component_name = CharField(max_length=50) + f_component_entrypoint = JSONField(null=True) + f_component_params = JSONField(null=True) + + class Meta: + db_table = "t_component_info" + primary_key = CompositeKey("f_provider_name", "f_component_name") + + +class PipelineModelMeta(DataBaseModel): + f_model_id = CharField(max_length=100) + f_model_version = CharField(max_length=10) + f_job_id = CharField(max_length=25, index=True) + f_role = CharField(max_length=50, index=True) + f_party_id = CharField(max_length=50, index=True) + f_task_name = CharField(max_length=50, index=True) + f_storage_key = CharField(max_length=100) + f_output_key = CharField(max_length=20) + f_type_name = CharField(max_length=20) + f_meta_data = JSONField(null=True) + f_storage_engine = CharField(max_length=30, null=True, index=True) + + class Meta: + db_table = 't_model_meta' + primary_key = CompositeKey('f_job_id', 'f_storage_key', "f_storage_engine") + + +class ServerRegistryInfo(DataBaseModel): + f_server_name = CharField(max_length=30, index=True) + f_host = CharField(max_length=30) + f_port = IntegerField() + f_protocol = CharField(max_length=10) + + class Meta: + db_table = "t_server" + + +class ServiceRegistryInfo(DataBaseModel): + f_server_name = CharField(max_length=30) + f_service_name = CharField(max_length=30) + f_url = CharField(max_length=100) + f_method = CharField(max_length=10) + f_params = JSONField(null=True) + f_data = JSONField(null=True) + f_headers = JSONField(null=True) + + class Meta: + db_table = "t_service" + primary_key = CompositeKey('f_server_name', 'f_service_name') diff --git a/python/fate_flow/db/permission_models.py b/python/fate_flow/db/permission_models.py new file mode 100644 index 000000000..090c15510 --- /dev/null +++ b/python/fate_flow/db/permission_models.py @@ -0,0 +1,37 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from peewee import CharField, CompositeKey + +from fate_flow.db import DataBaseModel + + +class AppInfo(DataBaseModel): + f_app_name = CharField(max_length=100, index=True) + f_app_id = CharField(max_length=100, primary_key=True) + f_app_token = CharField(max_length=100) + f_app_type = CharField(max_length=20, index=True) + + class Meta: + db_table = "t_app_info" + + +class PartnerAppInfo(DataBaseModel): + f_party_id = CharField(max_length=100, primary_key=True) + f_app_id = CharField(max_length=100) + f_app_token = CharField(max_length=100) + + class Meta: + db_table = "t_partner_app_info" diff --git a/python/fate_flow/db/schedule_models.py b/python/fate_flow/db/schedule_models.py index 8973d565d..14d974746 100644 --- a/python/fate_flow/db/schedule_models.py +++ b/python/fate_flow/db/schedule_models.py @@ -15,11 +15,12 @@ # from peewee import CharField, TextField, IntegerField, BooleanField, BigIntegerField, CompositeKey -from fate_flow.db.base_models import DataBaseModel, JSONField, DateTimeField +from fate_flow.db.base_models import DataBaseModel, JSONField class ScheduleJob(DataBaseModel): f_job_id = CharField(max_length=25, index=True) + f_priority = IntegerField(default=0) f_tag = CharField(max_length=50, null=True, default='') f_dag = JSONField(null=True) f_parties = JSONField() @@ -29,20 +30,15 @@ class ScheduleJob(DataBaseModel): f_status_code = IntegerField(null=True) f_progress = IntegerField(null=True, default=0) - f_ready_signal = BooleanField(default=False) - f_ready_time = BigIntegerField(null=True) + f_schedule_signal = BooleanField(default=False) + f_schedule_time = BigIntegerField(null=True) f_cancel_signal = BooleanField(default=False) f_cancel_time = BigIntegerField(null=True) f_rerun_signal = BooleanField(default=False) f_end_scheduling_updates = IntegerField(null=True, default=0) - f_inheritance_info = JSONField(null=True) - f_inheritance_status = CharField(max_length=50, null=True) - f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) f_elapsed = BigIntegerField(null=True) class Meta: @@ -63,9 +59,7 @@ class ScheduleTask(DataBaseModel): f_status = CharField(max_length=50) f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) f_elapsed = BigIntegerField(null=True) class Meta: @@ -80,7 +74,7 @@ class ScheduleTaskStatus(DataBaseModel): f_task_version = BigIntegerField() f_status = CharField(max_length=50) f_auto_retries = IntegerField(default=0) - f_federated_status_collect_type = CharField(max_length=10) + f_sync_type = CharField(max_length=10) class Meta: db_table = "t_schedule_task_status" diff --git a/python/fate_flow/db/storage_models.py b/python/fate_flow/db/storage_models.py index bc81100ab..f291ad686 100644 --- a/python/fate_flow/db/storage_models.py +++ b/python/fate_flow/db/storage_models.py @@ -14,7 +14,7 @@ # limitations under the License. from peewee import CharField, IntegerField, BooleanField, BigIntegerField, TextField, DateTimeField, CompositeKey -from fate_flow.db.base_models import DataBaseModel, JSONField, SerializedField +from fate_flow.db.base_models import DataBaseModel, JSONField class StorageConnectorModel(DataBaseModel): @@ -31,27 +31,22 @@ class StorageTableMetaModel(DataBaseModel): f_namespace = CharField(max_length=100, index=True) f_address = JSONField() f_engine = CharField(max_length=100) # 'EGGROLL', 'MYSQL' - f_store_type = CharField(max_length=50, null=True) # store type f_options = JSONField() f_partitions = IntegerField(null=True) f_delimiter = CharField(null=True) - f_in_serialized = BooleanField(default=True) f_have_head = BooleanField(default=True) f_extend_sid = BooleanField(default=False) - f_auto_increasing_sid = BooleanField(default=False) - - f_schema = JSONField() + f_data_meta = JSONField() f_count = BigIntegerField(null=True) f_part_of_data = JSONField() - f_origin = CharField(max_length=50, default='') + f_source = JSONField() + f_data_type = CharField(max_length=20, null=True) f_disable = BooleanField(default=False) f_description = TextField(default='') f_read_access_time = BigIntegerField(null=True) - f_read_access_date = DateTimeField(null=True) f_write_access_time = BigIntegerField(null=True) - f_write_access_date = DateTimeField(null=True) class Meta: db_table = "t_storage_table_meta" diff --git a/python/fate_flow/detection/detector.py b/python/fate_flow/detection/detector.py index e2af121c9..410f8f753 100644 --- a/python/fate_flow/detection/detector.py +++ b/python/fate_flow/detection/detector.py @@ -15,8 +15,8 @@ # import time -from fate_flow.engine.computing import build_engine -from fate_flow.entity.run_status import TaskStatus, JobStatus +from fate_flow.engine.devices import build_engine +from fate_flow.entity.types import TaskStatus, JobStatus from fate_flow.operation.job_saver import JobSaver from fate_flow.runtime.runtime_config import RuntimeConfig from fate_flow.scheduler.federated_scheduler import FederatedScheduler @@ -37,7 +37,7 @@ def detect_running_task(cls): count = 0 try: running_tasks = JobSaver.query_task(party_status=TaskStatus.RUNNING) - detect_logger().info(f'running task test: {running_tasks}') + detect_logger().info(f'running task: {running_tasks}') stop_job_ids = set() for task in running_tasks: if task.f_run_ip != RuntimeConfig.JOB_SERVER_HOST: @@ -45,7 +45,7 @@ def detect_running_task(cls): continue count += 1 try: - process_exist = build_engine().is_alive(task) + process_exist = build_engine(task.f_provider_name).is_alive(task) if not process_exist: msg = f"task {task.f_task_id} {task.f_task_version} on {task.f_role} {task.f_party_id}" detect_logger(job_id=task.f_job_id).info( @@ -107,7 +107,43 @@ def request_stop_jobs(cls, jobs, stop_msg, stop_status): @classmethod def detect_cluster_instance_status(cls, task, stop_job_ids): - pass + detect_logger(job_id=task.f_job_id).info('start detect running task instance status') + try: + latest_tasks = JobSaver.query_task(task_id=task.f_task_id, role=task.f_role, party_id=task.f_party_id) + + if len(latest_tasks) != 1: + detect_logger(job_id=task.f_job_id).error( + f'query latest tasks of {task.f_task_id} failed, ' + f'have {len(latest_tasks)} tasks' + ) + return + + if task.f_task_version != latest_tasks[0].f_task_version: + detect_logger(job_id=task.f_job_id).info( + f'{task.f_task_id} {task.f_task_version} is not the latest task, ' + 'update task status to failed' + ) + JobSaver.update_task_status({ + 'task_id': task.f_task_id, + 'role': task.f_role, + 'party_id': task.f_party_id, + 'task_version': task.f_task_version, + 'status': JobStatus.FAILED, + 'party_status': JobStatus.FAILED, + }) + return + + instance_list = RuntimeConfig.SERVICE_DB.get_servers() + instance_list = {instance.http_address for instance_id, instance in instance_list.items()} + + if f'{task.f_run_ip}:{task.f_run_port}' not in instance_list: + detect_logger(job_id=task.f_job_id).error( + 'detect cluster instance status failed, ' + 'add task {task.f_task_id} {task.f_task_version} to stop list' + ) + stop_job_ids.add(task.f_job_id) + except Exception as e: + detect_logger(job_id=task.f_job_id).exception(e) class FederatedDetector(Detector): diff --git a/python/fate_flow/engine/computing/__init__.py b/python/fate_flow/engine/backend/__init__.py similarity index 88% rename from python/fate_flow/engine/computing/__init__.py rename to python/fate_flow/engine/backend/__init__.py index 3154760cd..90057cf46 100644 --- a/python/fate_flow/engine/computing/__init__.py +++ b/python/fate_flow/engine/backend/__init__.py @@ -12,10 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from fate_flow.engine.computing._session import build_engine +from fate_flow.engine.backend._session import build_backend __all__ = [ - "build_engine", + "build_backend", ] diff --git a/python/fate_flow/engine/backend/_base.py b/python/fate_flow/engine/backend/_base.py new file mode 100644 index 000000000..f98b10ad3 --- /dev/null +++ b/python/fate_flow/engine/backend/_base.py @@ -0,0 +1,150 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import abc +import logging +import os +import sys +import typing + +import yaml + +from fate_flow.db.db_models import Task +from fate_flow.entity.types import ProviderName, WorkerName +from fate_flow.manager.service.worker_manager import WorkerManager +from fate_flow.utils.job_utils import get_task_directory + + +class EngineABC(metaclass=abc.ABCMeta): + @abc.abstractmethod + def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs) -> typing.Dict: + ... + + @abc.abstractmethod + def kill(self, task: Task): + ... + + @abc.abstractmethod + def is_alive(self, task: Task): + ... + + +class LocalEngine(object): + @classmethod + def get_component_define(cls, provider_name, task_info, stage): + task_dir = get_task_directory(**task_info, output=True) + component_ref = task_info.get("component") + role = task_info.get("role") + os.makedirs(task_dir, exist_ok=True) + define_file = os.path.join(task_dir, "define.yaml") + cmd = cls.generate_component_define_cmd(provider_name, component_ref, role, stage, define_file) + logging.debug(f"load define cmd: {cmd}") + if cmd: + WorkerManager.start_task_worker( + worker_name=WorkerName.COMPONENT_DEFINE, + task_info=task_info, + common_cmd=cmd, + sync=True + ) + if os.path.exists(define_file): + with open(define_file, "r") as fr: + return yaml.safe_load(fr) + return {} + + def _cleanup1(self, **kwargs): + # backend cleanup + pass + + def _cleanup2(self, provider_name, task_info, config, **kwargs): + # engine cleanup: computing、federation .. + cmd = self.generate_cleanup_cmd(provider_name) + + if cmd: + logging.info(f"start clean task, config: {config}") + WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_EXECUTE_CLEAN, + task_info=task_info, + common_cmd=cmd, + task_parameters=config, + sync=True + ) + logging.info(f"clean success") + + def cleanup(self, provider_name, task_info, config, party_task_id, **kwargs): + self._cleanup1(session_id=party_task_id, task_info=task_info) + self._cleanup2(provider_name, task_info, config, **kwargs) + + @staticmethod + def generate_component_run_cmd(provider_name, conf_path, output_path=""): + if provider_name == ProviderName.FATE: + from fate_flow.worker.fate_executor import FateSubmit + module_file_path = sys.modules[FateSubmit.__module__].__file__ + + elif provider_name == ProviderName.FATE_FLOW: + from fate_flow.worker.fate_flow_executor import FateFlowSubmit + module_file_path = sys.modules[FateFlowSubmit.__module__].__file__ + + else: + raise ValueError(f"load provider {provider_name} failed") + os.environ.pop("FATE_TASK_CONFIG", None) + common_cmd = [ + module_file_path, + "component", + "execute", + "--config", + conf_path, + "--execution-final-meta-path", + output_path + ] + + return common_cmd + + @staticmethod + def generate_component_define_cmd(provider_name, component_ref, role, stage, define_file): + cmd = [] + if provider_name == ProviderName.FATE: + from fate_flow.worker.fate_executor import FateSubmit + module_file_path = sys.modules[FateSubmit.__module__].__file__ + cmd = [ + module_file_path, + "component", + "artifact-type", + "--name", + component_ref, + "--role", + role, + "--stage", + stage, + "--output-path", + define_file + ] + return cmd + + @staticmethod + def generate_cleanup_cmd(provider_name): + cmd = [] + if provider_name == ProviderName.FATE: + from fate_flow.worker.fate_executor import FateSubmit + module_file_path = sys.modules[FateSubmit.__module__].__file__ + cmd = [ + module_file_path, + "component", + "cleanup", + "--env-name", + "FATE_TASK_CONFIG", + ] + return cmd diff --git a/python/fate_flow/engine/backend/_eggroll.py b/python/fate_flow/engine/backend/_eggroll.py new file mode 100644 index 000000000..50a1caf75 --- /dev/null +++ b/python/fate_flow/engine/backend/_eggroll.py @@ -0,0 +1,38 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml + +from fate_flow.engine.backend._base import LocalEngine +from fate_flow.entity.spec.dag import TaskConfigSpec +from fate_flow.entity.types import WorkerName, ComputingEngine +from fate_flow.manager.service.worker_manager import WorkerManager + + +class EggrollEngine(LocalEngine): + def run(self, task_info, run_parameters, engine_run, provider_name, output_path, conf_path, sync=False, **kwargs): + parameters = TaskConfigSpec.parse_obj(run_parameters) + if parameters.conf.computing.type == ComputingEngine.EGGROLL: + # update eggroll options + parameters.conf.computing.metadata.options.update(engine_run) + with open(conf_path, "w") as f: + # update parameters + yaml.dump(parameters.dict(), f) + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_EXECUTE, + task_info=task_info, + common_cmd=self.generate_component_run_cmd(provider_name, conf_path, output_path, ), + sync=sync + ) diff --git a/python/fate_flow/engine/backend/_session.py b/python/fate_flow/engine/backend/_session.py new file mode 100644 index 000000000..41e0da08e --- /dev/null +++ b/python/fate_flow/engine/backend/_session.py @@ -0,0 +1,34 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.backend._eggroll import EggrollEngine +from fate_flow.engine.backend._spark import SparkEngine +from fate_flow.engine.backend.eggroll_deepspeed import EggrollDeepspeedEngine +from fate_flow.entity.types import ComputingEngine, LauncherType + + +def build_backend(backend_name: str, launcher_name: str = LauncherType.DEFAULT): + if backend_name in {ComputingEngine.EGGROLL, ComputingEngine.STANDALONE}: + if launcher_name == LauncherType.DEEPSPEED: + backend = EggrollDeepspeedEngine() + elif not launcher_name or launcher_name == LauncherType.DEFAULT: + backend = EggrollEngine() + else: + raise ValueError(f'backend "{backend_name}" launcher {launcher_name} is not supported') + elif backend_name == ComputingEngine.SPARK: + backend = SparkEngine() + else: + raise ValueError(f'backend "{backend_name}" is not supported') + return backend diff --git a/python/fate_flow/engine/backend/_spark.py b/python/fate_flow/engine/backend/_spark.py new file mode 100644 index 000000000..5d49f2b4f --- /dev/null +++ b/python/fate_flow/engine/backend/_spark.py @@ -0,0 +1,56 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from fate_flow.engine.backend._base import LocalEngine +from fate_flow.entity.types import WorkerName +from fate_flow.manager.service.worker_manager import WorkerManager + + +class SparkEngine(LocalEngine): + def run(self, task_info, run_parameters, conf_path, output_path, engine_run, provider_name, **kwargs): + spark_home = os.environ.get("SPARK_HOME", None) + if not spark_home: + try: + import pyspark + spark_home = pyspark.__path__[0] + except ImportError as e: + raise RuntimeError("can not import pyspark") + except Exception as e: + raise RuntimeError("can not import pyspark") + + deploy_mode = engine_run.get("deploy-mode", "client") + if deploy_mode not in ["client"]: + raise ValueError(f"deploy mode {deploy_mode} not supported") + + spark_submit_cmd = os.path.join(spark_home, "bin/spark-submit") + process_cmd = [spark_submit_cmd, f"--name={task_info.get('task_id')}#{task_info.get('role')}"] + for k, v in engine_run.items(): + if k != "conf": + process_cmd.append(f"--{k}={v}") + if "conf" in engine_run: + for ck, cv in engine_run["conf"].items(): + process_cmd.append(f"--conf") + process_cmd.append(f"{ck}={cv}") + extra_env = {"SPARK_HOME": spark_home} + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_EXECUTE, + task_info=task_info, + common_cmd=self.generate_component_run_cmd(provider_name, conf_path, output_path), + extra_env=extra_env, + executable=process_cmd, + sync=True + ) diff --git a/python/fate_flow/engine/backend/eggroll_deepspeed.py b/python/fate_flow/engine/backend/eggroll_deepspeed.py new file mode 100644 index 000000000..dd8f2c395 --- /dev/null +++ b/python/fate_flow/engine/backend/eggroll_deepspeed.py @@ -0,0 +1,225 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import logging +import os +import sys +import time +import traceback + +from fate_flow.engine.backend._base import LocalEngine +from fate_flow.entity.types import BaseStatus, TaskStatus +from fate_flow.utils import job_utils +from fate_flow.worker.fate_executor import FateSubmit + +logger = logging.getLogger(__name__) + + +class StatusSet(BaseStatus): + NEW = "NEW" + NEW_TIMEOUT = "NEW_TIMEOUT" + ACTIVE = "ACTIVE" + CLOSED = "CLOSED" + KILLED = "KILLED" + ERROR = "ERROR" + FINISHED = "FINISHED" + + +class EndStatus(BaseStatus): + NEW_TIMEOUT = StatusSet.NEW_TIMEOUT + CLOSED = StatusSet.CLOSED + FAILED = StatusSet.KILLED + ERROR = StatusSet.ERROR + FINISHED = StatusSet.FINISHED + + +class EggrollDeepspeedEngine(LocalEngine): + @staticmethod + def generate_session_id(): + return f"deepspeed_session_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" + + def run(self, output_path, conf_path, session_id, task_info, launcher_conf, **kwargs): + logger.info("deepspeed task start") + command_arguments = self.generate_command_arguments(conf_path, output_path) + + resource_options = {"timeout_seconds": launcher_conf.get("timeout"), "resource_exhausted_strategy": "waiting"} + options = {"eggroll.container.deepspeed.script.path": sys.modules[FateSubmit.__module__].__file__} + world_size = launcher_conf.get("world_size") + logger.info(f"command_arguments: {command_arguments}\n resource_options: {resource_options}\n " + f"options: {options}\n world_size: {world_size}") + + from eggroll.deepspeed.submit import client + # set session id == party task id + client = client.DeepspeedJob(session_id) + + resp = client.submit( + world_size=world_size, + command_arguments=command_arguments, + environment_variables={}, + files={}, + resource_options=resource_options, + options=options + ) + logger.info(f"submit deepspeed {resp.session_id} task success") + + status = self.wait_deepspeed_job(session_id=session_id, timeout=launcher_conf.get("timeout")) + logger.info(f"deepspeed task end with status {status}") + if status not in EndStatus.status_list(): + logger.info(f"start to kill deepspeed task {session_id}") + self.kill(session_id=session_id) + + # download logs and models + self.download_to(session_id, task_info) + + def wait_deepspeed_job(self, session_id, timeout): + while True: + status = self.query_status(session_id=session_id) + if timeout % 10 == 0: + logger.info(f"deepspeed task status {status}") + timeout -= 1 + if timeout == 0: + return status + elif status in EndStatus.status_list(): + return status + time.sleep(2) + + @staticmethod + def generate_command_arguments(conf_path, output_path=""): + command_arguments = [ + "component", + "execute", + "---config", + conf_path, + "FATE_TASK_CONFIG", + "--execution-final-meta-path", + output_path + ] + return command_arguments + + def _cleanup1(self, session_id, task_info, **kwargs): + self.kill(session_id) + self.download_to(session_id, task_info) + + @staticmethod + def kill(session_id): + if session_id: + logger.info(f"start kill deepspeed task {session_id}") + from eggroll.deepspeed.submit import client + client = client.DeepspeedJob(session_id) + try: + client.kill() + except Exception as e: + traceback.format_exc() + logger.error(e) + + @staticmethod + def _query_status(session_id): + if session_id: + from eggroll.deepspeed.submit import client + client = client.DeepspeedJob(session_id) + _s = client.query_status().status + return _s if _s else StatusSet.NEW + return StatusSet.NEW + + @staticmethod + def _download_job(session_id, base_dir, content_type=None, ranks: list = None): + from eggroll.deepspeed.submit import client + if not content_type: + content_type = client.ContentType.ALL + if session_id: + client = client.DeepspeedJob(session_id) + os.makedirs(base_dir, exist_ok=True) + path = lambda rank: f"{base_dir}/{rank}.zip" + client.download_job_to(rank_to_path=path, content_type=content_type, ranks=ranks) + return base_dir + + def query_status(self, session_id): + status = self._query_status(session_id) + if status in EndStatus.status_list(): + if status in [EndStatus.FINISHED]: + return TaskStatus.SUCCESS + else: + return TaskStatus.FAILED + + def is_alive(self, task): + status = self._query_status(task) + if status in StatusSet.status_list(): + if status in EndStatus.status_list(): + return False + else: + return True + else: + raise RuntimeError(f"task run status: {status}") + + def download(self, session_id, base_dir, content_type=None, ranks=None): + from eggroll.deepspeed.submit.client import ContentType + if not content_type: + content_type = ContentType.ALL + dir_name = self._download_job(session_id, base_dir, content_type, ranks) + if dir_name: + for file in os.listdir(dir_name): + if file.endswith(".zip"): + rank_dir = os.path.join(dir_name, file.split(".zip")[0]) + os.makedirs(rank_dir, exist_ok=True) + self.unzip(os.path.join(dir_name, file), extra_dir=rank_dir) + os.remove(os.path.join(dir_name, file)) + + def download_to(self, session_id, task_info): + try: + logger.info(f"end task") + path = self.download_model(session_id=session_id, task_info=task_info) + logger.info(f"download model to {path}") + path = self.download_log(session_id=session_id, task_info=task_info) + logger.info(f"download logs to {path}") + except Exception as e: + traceback.format_exc() + logger.error(e) + + def download_log(self, session_id, task_info, path=None): + from eggroll.deepspeed.submit.client import ContentType + if not path: + path = self.log_path(task_info) + self.download(session_id, base_dir=path, content_type=ContentType.LOGS) + return path + + def download_model(self, session_id, task_info, path=None): + from eggroll.deepspeed.submit.client import ContentType + if not path: + path = self.model_path(task_info) + self.download(session_id, base_dir=path, content_type=ContentType.MODELS, ranks=[0]) + return path + + @staticmethod + def unzip(zip_path, extra_dir): + import zipfile + zfile = zipfile.ZipFile(zip_path, "r") + for name in zfile.namelist(): + dir_name = os.path.dirname(zip_path) + file_path = os.path.join(dir_name, extra_dir, name) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + data = zfile.read(name) + with open(file_path, "w+b") as file: + file.write(data) + + @staticmethod + def model_path(task_info,): + return os.path.join(job_utils.get_task_directory(**task_info, output=True), "model") + + @staticmethod + def log_path(task_info): + return job_utils.get_job_log_directory( + task_info.get("job_id"), task_info.get("role"), task_info.get("party_id"), task_info.get("task_name") + ) diff --git a/python/fate_flow/engine/computing/_eggroll.py b/python/fate_flow/engine/computing/_eggroll.py deleted file mode 100644 index ac10537e0..000000000 --- a/python/fate_flow/engine/computing/_eggroll.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.db.db_models import Task -from fate_flow.engine.computing._base import EngineABC -from fate_flow.entity.run_status import TaskStatus -from fate_flow.entity.types import KillProcessRetCode, WorkerName -from fate_flow.manager.containerd_worker_manager import ContainerdWorkerManager -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.utils import job_utils, process_utils - - -class EggrollEngine(EngineABC): - def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): - return WorkerManager.start_task_worker(worker_name=WorkerName.TASK_EXECUTOR, task=task, - task_parameters=run_parameters) - - def kill(self, task): - kill_status_code = process_utils.kill_task_executor_process(task) - # session stop - if kill_status_code is KillProcessRetCode.KILLED or task.f_status not in {TaskStatus.WAITING}: - job_utils.start_session_stop(task) - - def is_alive(self, task): - return process_utils.check_process(pid=int(task.f_run_pid), task=task) - - -class ContainerdEggrollEngine(ContainerdWorkerManager): - pass diff --git a/python/fate_flow/engine/computing/_session.py b/python/fate_flow/engine/computing/_session.py deleted file mode 100644 index 36cdd46b5..000000000 --- a/python/fate_flow/engine/computing/_session.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.engine.computing._eggroll import ContainerdEggrollEngine, EggrollEngine -from fate_flow.engine.computing._spark import SparkEngine -from fate_flow.entity.engine_types import ComputingEngine, EngineType -from fate_flow.settings import ENGINES, WORKER - - -def build_engine(computing_engine=None): - if not computing_engine: - computing_engine = ENGINES.get(EngineType.COMPUTING) - if computing_engine in {ComputingEngine.EGGROLL, ComputingEngine.STANDALONE}: - if WORKER.get('type') in {'docker', 'k8s'}: - engine_session = ContainerdEggrollEngine() - else: - engine_session = EggrollEngine() - elif computing_engine == ComputingEngine.SPARK: - engine_session = SparkEngine() - else: - raise ValueError(f'engine "{computing_engine}" is not supported') - return engine_session diff --git a/python/fate_flow/engine/computing/_spark.py b/python/fate_flow/engine/computing/_spark.py deleted file mode 100644 index 2ff4d4da9..000000000 --- a/python/fate_flow/engine/computing/_spark.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os - -from fate_flow.db.db_models import Task -from fate_flow.engine.computing._base import EngineABC -from fate_flow.entity.run_status import TaskStatus -from fate_flow.entity.types import KillProcessRetCode, WorkerName -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.utils import job_utils, process_utils - - -class SparkEngine(EngineABC): - def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): - # todo: get spark home from server registry - spark_home = None - if not spark_home: - try: - import pyspark - spark_home = pyspark.__path__[0] - except ImportError as e: - raise RuntimeError("can not import pyspark") - except Exception as e: - raise RuntimeError("can not import pyspark") - # else: - # raise ValueError(f"spark home must be configured in conf/service_conf.yaml when run on cluster mode") - - # additional configs - spark_submit_config = run_parameters.get("conf", {}).get("computing", {}).get("metadata", {}).get("spark_run", {}) - - deploy_mode = spark_submit_config.get("deploy-mode", "client") - if deploy_mode not in ["client"]: - raise ValueError(f"deploy mode {deploy_mode} not supported") - - spark_submit_cmd = os.path.join(spark_home, "bin/spark-submit") - process_cmd = [spark_submit_cmd, f"--name={task.f_task_id}#{task.f_role}"] - for k, v in spark_submit_config.items(): - if k != "conf": - process_cmd.append(f"--{k}={v}") - if "conf" in spark_submit_config: - for ck, cv in spark_submit_config["conf"].items(): - process_cmd.append(f"--conf") - process_cmd.append(f"{ck}={cv}") - extra_env = {"SPARK_HOME": spark_home} - return WorkerManager.start_task_worker(worker_name=WorkerName.TASK_EXECUTOR, task=task, - task_parameters=run_parameters, - extra_env=extra_env, executable=process_cmd) - - def kill(self, task): - kill_status_code = process_utils.kill_task_executor_process(task) - # session stop - if kill_status_code is KillProcessRetCode.KILLED or task.f_status not in {TaskStatus.WAITING}: - job_utils.start_session_stop(task) - - def is_alive(self, task): - return process_utils.check_process(pid=int(task.f_run_pid), task=task) diff --git a/python/fate_flow/engine/devices/__init__.py b/python/fate_flow/engine/devices/__init__.py new file mode 100644 index 000000000..a88fd885c --- /dev/null +++ b/python/fate_flow/engine/devices/__init__.py @@ -0,0 +1,34 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.entity.types import ProviderDevice +from fate_flow.manager.service.provider_manager import ProviderManager + + +def build_engine(provider_name: str): + provider = ProviderManager.get_provider_by_provider_name(provider_name) + + if provider.device in {ProviderDevice.DOCKER, ProviderDevice.K8S}: + from fate_flow.engine.devices._container import ContainerdEngine + engine_session = ContainerdEngine(provider) + + elif provider.device in {ProviderDevice.LOCAL}: + from fate_flow.engine.devices._local import LocalEngine + engine_session = LocalEngine(provider) + + else: + raise ValueError(f'engine device "{provider.device}" is not supported') + + return engine_session + diff --git a/python/fate_flow/engine/computing/_base.py b/python/fate_flow/engine/devices/_base.py similarity index 88% rename from python/fate_flow/engine/computing/_base.py rename to python/fate_flow/engine/devices/_base.py index dc781c9ac..8b7a5c7ba 100644 --- a/python/fate_flow/engine/computing/_base.py +++ b/python/fate_flow/engine/devices/_base.py @@ -16,9 +16,11 @@ import abc +import sys import typing from fate_flow.db.db_models import Task +from fate_flow.entity.types import ProviderName class EngineABC(metaclass=abc.ABCMeta): @@ -33,3 +35,7 @@ def kill(self, task: Task): @abc.abstractmethod def is_alive(self, task: Task): ... + + @abc.abstractmethod + def cleanup(self, task: Task): + ... diff --git a/python/fate_flow/manager/containerd_worker_manager.py b/python/fate_flow/engine/devices/_container.py similarity index 60% rename from python/fate_flow/manager/containerd_worker_manager.py rename to python/fate_flow/engine/devices/_container.py index 259eebbd2..214660241 100644 --- a/python/fate_flow/manager/containerd_worker_manager.py +++ b/python/fate_flow/engine/devices/_container.py @@ -13,31 +13,32 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from ruamel import yaml +import yaml from fate_flow.db.db_models import Task +from fate_flow.engine.devices._base import EngineABC +from fate_flow.entity.types import ProviderDevice from fate_flow.runtime.runtime_config import RuntimeConfig -from fate_flow.settings import WORKER from fate_flow.utils.log_utils import schedule_logger -class ContainerdWorkerManager: - worker_type = WORKER.get('type', '') - - def __init__(self): - if self.worker_type == 'docker': - from fate_flow.manager.docker_manager import DockerManager - self.manager = DockerManager() - elif self.worker_type == 'k8s': - from fate_flow.manager.k8s_manager import K8sManager - self.manager = K8sManager() +class ContainerdEngine(EngineABC): + def __init__(self, provider): + if provider.device == ProviderDevice.DOCKER: + from fate_flow.manager.container.docker_manager import DockerManager + self.manager = DockerManager(provider) + elif provider.device == ProviderDevice.K8S: + from fate_flow.manager.container.k8s_manager import K8sManager + self.manager = K8sManager(provider) else: - raise ValueError(f'worker "{self.worker_type}" is not supported') + raise ValueError(f'worker "{provider.device}" is not supported') - def get_name(self, task: Task): + @staticmethod + def _get_name(task: Task): return f'{task.f_role}-{task.f_party_id}-{task.f_task_id}-{task.f_task_version}' - def get_command(self, task: Task): + @staticmethod + def _get_command(task: Task): return [ '-m', 'fate.components', @@ -49,16 +50,17 @@ def get_command(self, task: Task): 'FATE_TASK_CONFIG', ] - def get_environment(self, task: Task, run_parameters): + @staticmethod + def _get_environment(task: Task, run_parameters): return { 'FATE_JOB_ID': task.f_job_id, 'FATE_TASK_CONFIG': yaml.dump(run_parameters), } def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): - name = self.get_name(task) - cmd = self.get_command(task) - env = self.get_environment(task, run_parameters) + name = self._get_name(task) + cmd = self._get_command(task) + env = self._get_environment(task, run_parameters) schedule_logger(job_id=task.f_job_id).info(f"start run container {name}, cmd: {cmd}, env: {env}") self.manager.start(name, cmd, env) return { @@ -67,7 +69,11 @@ def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_d } def kill(self, task: Task): - self.manager.stop(self.get_name(task)) + self.manager.stop(self._get_name(task)) def is_alive(self, task: Task): - return self.manager.is_running(self.get_name(task)) + return self.manager.is_running(self._get_name(task)) + + def cleanup(self, task: Task): + pass + diff --git a/python/fate_flow/engine/devices/_local.py b/python/fate_flow/engine/devices/_local.py new file mode 100644 index 000000000..4891bc369 --- /dev/null +++ b/python/fate_flow/engine/devices/_local.py @@ -0,0 +1,82 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import sys + +from fate_flow.db.db_models import Task +from fate_flow.engine.devices._base import EngineABC +from fate_flow.entity.types import TaskStatus, WorkerName, ProviderName +from fate_flow.entity.code import KillProcessRetCode +from fate_flow.manager.service.worker_manager import WorkerManager +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.utils import job_utils, process_utils + + +class LocalEngine(EngineABC): + def __init__(self, provider: ComponentProvider): + self.provider = provider + + def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_ENTRYPOINT, + task_info=task.to_human_model_dict(), + extra_env={"PYTHONPATH": self.provider.python_path}, + executable=[self.provider.python_env], + common_cmd=self.generate_cmd(), + task_parameters=run_parameters, + record=True + ) + + def kill(self, task): + process_utils.kill_task_executor_process(task) + + def is_alive(self, task): + return process_utils.check_process(pid=int(task.f_run_pid), task=task) + + def cleanup(self, task: Task): + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_CLEAN, + task_info=task.to_human_model_dict(), + extra_env={"PYTHONPATH": self.provider.python_path}, + executable=[self.provider.python_env], + common_cmd=self.generate_cleanup_cmd(), + task_parameters=task.f_component_parameters + ) + + @staticmethod + def generate_cmd(): + from fate_flow.entrypoint.runner import Submit + module_file_path = sys.modules[Submit.__module__].__file__ + common_cmd = [ + module_file_path, + "component", + "entrypoint", + "--env-name", + "FATE_TASK_CONFIG", + ] + return common_cmd + + @staticmethod + def generate_cleanup_cmd(): + from fate_flow.entrypoint.runner import Submit + module_file_path = sys.modules[Submit.__module__].__file__ + common_cmd = [ + module_file_path, + "component", + "cleanup", + "--env-name", + "FATE_TASK_CONFIG", + ] + return common_cmd diff --git a/python/fate_flow/engine/relation_ship.py b/python/fate_flow/engine/relation_ship.py index 8e79cd7db..3693f0548 100644 --- a/python/fate_flow/engine/relation_ship.py +++ b/python/fate_flow/engine/relation_ship.py @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from fate_flow.entity.address_types import StandaloneAddress, EggRollAddress, HDFSAddress, MysqlAddress, HiveAddress, LocalFSAddress, PathAddress, \ - ApiAddress -from fate_flow.entity.engine_types import ComputingEngine, StorageEngine, FederationEngine, EngineType +from fate_flow.entity.types import StandaloneAddress, EggRollAddress, HDFSAddress, MysqlAddress, HiveAddress, \ + PathAddress, ApiAddress, ComputingEngine, StorageEngine, FederationEngine, EngineType, FileAddress class Relationship(object): @@ -57,12 +56,13 @@ class Relationship(object): "support": [ StorageEngine.HDFS, StorageEngine.HIVE, - StorageEngine.LOCALFS, + StorageEngine.FILE, + StorageEngine.STANDALONE ], }, EngineType.FEDERATION: { "default": FederationEngine.RABBITMQ, - "support": [FederationEngine.PULSAR, FederationEngine.RABBITMQ, FederationEngine.OSX], + "support": [FederationEngine.PULSAR, FederationEngine.RABBITMQ, FederationEngine.OSX, FederationEngine.STANDALONE], }, } } @@ -73,7 +73,7 @@ class Relationship(object): StorageEngine.HDFS: HDFSAddress, StorageEngine.MYSQL: MysqlAddress, StorageEngine.HIVE: HiveAddress, - StorageEngine.LOCALFS: LocalFSAddress, + StorageEngine.FILE: FileAddress, StorageEngine.PATH: PathAddress, StorageEngine.API: ApiAddress } diff --git a/python/fate_flow/engine/storage/__init__.py b/python/fate_flow/engine/storage/__init__.py index 59569702b..f2e4f13dd 100644 --- a/python/fate_flow/engine/storage/__init__.py +++ b/python/fate_flow/engine/storage/__init__.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from fate_flow.engine.storage._types import EggRollStoreType, StorageEngine, StandaloneStoreType, StorageTableOrigin +from fate_flow.engine.storage._types import EggRollStoreType, StorageEngine, StandaloneStoreType, DataType, StorageOrigin from fate_flow.engine.storage._table import StorageTableBase, StorageTableMeta from fate_flow.engine.storage._session import StorageSessionBase, Session - - diff --git a/python/fate_flow/engine/abc/_storage.py b/python/fate_flow/engine/storage/_abc.py similarity index 93% rename from python/fate_flow/engine/abc/_storage.py rename to python/fate_flow/engine/storage/_abc.py index e6ce7e720..ffc83e041 100644 --- a/python/fate_flow/engine/abc/_storage.py +++ b/python/fate_flow/engine/storage/_abc.py @@ -30,7 +30,7 @@ def query_table_meta(self, filter_fields, query_fields=None): ... @abc.abstractmethod - def update_metas(self, schema=None, count=None, part_of_data=None, description=None, partitions=None, **kwargs): + def update_metas(self, data_meta=None, count=None, part_of_data=None, description=None, partitions=None, **kwargs): ... @abc.abstractmethod @@ -66,10 +66,6 @@ def get_options(self): def get_partitions(self): ... - @abc.abstractmethod - def get_in_serialized(self): - ... - @abc.abstractmethod def get_id_delimiter(self): ... @@ -87,7 +83,7 @@ def get_have_head(self): ... @abc.abstractmethod - def get_schema(self): + def get_data_meta(self): ... @abc.abstractmethod @@ -103,7 +99,7 @@ def get_description(self): ... @abc.abstractmethod - def get_origin(self): + def get_source(self): ... @abc.abstractmethod @@ -138,17 +134,17 @@ def engine(self): @property @abc.abstractmethod - def store_type(self): + def options(self): ... @property @abc.abstractmethod - def options(self): + def partitions(self): ... @property @abc.abstractmethod - def partitions(self): + def data_type(self): ... @property @@ -179,10 +175,6 @@ def create_meta(self, **kwargs) -> StorageTableMetaABC: def put_all(self, kv_list: Iterable, **kwargs): ... - @abc.abstractmethod - def put_meta(self, kv_list: Iterable, **kwargs): - ... - @abc.abstractmethod def collect(self, **kwargs) -> list: ... diff --git a/python/fate_flow/engine/storage/_session.py b/python/fate_flow/engine/storage/_session.py index 32fe87c8b..d3873f443 100644 --- a/python/fate_flow/engine/storage/_session.py +++ b/python/fate_flow/engine/storage/_session.py @@ -20,11 +20,11 @@ from fate_flow.db.base_models import DB from fate_flow.db.storage_models import SessionRecord -from fate_flow.engine.abc import StorageSessionABC, StorageTableABC, StorageTableMetaABC +from fate_flow.engine.storage._abc import StorageSessionABC, StorageTableABC, StorageTableMetaABC from fate_flow.engine.storage._table import StorageTableMeta -from fate_flow.entity.engine_types import EngineType, StorageEngine -from fate_flow.settings import ENGINES +from fate_flow.entity.types import EngineType, StorageEngine +from fate_flow.runtime.system_settings import ENGINES from fate_flow.utils import base_utils from fate_flow.utils.log import getLogger @@ -166,13 +166,19 @@ def _get_or_create_storage(self, if storage_engine == StorageEngine.EGGROLL: from fate_flow.engine.storage.eggroll import StorageSession - storage_session = StorageSession(session_id=storage_session_id, options=kwargs.get("options", {})) elif storage_engine == StorageEngine.STANDALONE: from fate_flow.engine.storage.standalone import StorageSession - storage_session = StorageSession(session_id=storage_session_id, options=kwargs.get("options", {})) + + elif storage_engine == StorageEngine.FILE: + from fate_flow.engine.storage.file import StorageSession + + elif storage_engine == StorageEngine.HDFS: + from fate_flow.engine.storage.hdfs import StorageSession + else: raise NotImplementedError(f"can not be initialized with storage engine: {storage_engine}") + storage_session = StorageSession(session_id=storage_session_id, options=kwargs.get("options", {})) self._storage_session[storage_session_id] = storage_session diff --git a/python/fate_flow/engine/storage/_table.py b/python/fate_flow/engine/storage/_table.py index 2db64f9da..8d095a694 100644 --- a/python/fate_flow/engine/storage/_table.py +++ b/python/fate_flow/engine/storage/_table.py @@ -22,10 +22,10 @@ from fate_flow.db.base_models import DB from fate_flow.db.storage_models import StorageTableMetaModel -from fate_flow.engine.abc import StorageTableMetaABC, StorageTableABC +from fate_flow.engine.storage._abc import StorageTableMetaABC, StorageTableABC from fate_flow.engine.relation_ship import Relationship -from fate_flow.entity.address_types import AddressABC +from fate_flow.entity.types import AddressABC from fate_flow.utils.base_utils import current_timestamp from fate_flow.utils.log import getLogger @@ -33,14 +33,13 @@ class StorageTableBase(StorageTableABC): - def __init__(self, name, namespace, address, partitions, options, engine, store_type): + def __init__(self, name, namespace, address, partitions, options, engine): self._name = name self._namespace = namespace self._address = address self._partitions = partitions self._options = options if options else {} self._engine = engine - self._store_type = store_type self._meta = None self._read_access_time = None @@ -50,10 +49,6 @@ def __init__(self, name, namespace, address, partitions, options, engine, store_ def name(self): return self._name - @property - def meta_name(self): - return f"{self.name}.meta" - @property def namespace(self): return self._namespace @@ -66,6 +61,10 @@ def address(self): def partitions(self): return self._partitions + @property + def data_type(self): + return self.meta.data_type + @property def options(self): return self._options @@ -74,10 +73,6 @@ def options(self): def engine(self): return self._engine - @property - def store_type(self): - return self._store_type - @property def meta(self): return self._meta @@ -95,13 +90,13 @@ def write_access_time(self): return self._write_access_time def update_meta(self, - schema=None, + data_meta=None, count=None, part_of_data=None, description=None, partitions=None, **kwargs): - self._meta.update_metas(schema=schema, + self._meta.update_metas(data_meta=data_meta, count=count, part_of_data=part_of_data, description=description, @@ -109,18 +104,25 @@ def update_meta(self, **kwargs) def create_meta(self, **kwargs): + self.destroy_if_exists() table_meta = StorageTableMeta(name=self._name, namespace=self._namespace, new=True) table_meta.set_metas(**kwargs) table_meta.address = self._address table_meta.partitions = self._partitions table_meta.engine = self._engine - table_meta.store_type = self._store_type table_meta.options = self._options table_meta.create() self._meta = table_meta return table_meta + def destroy_if_exists(self): + table_meta = StorageTableMeta(name=self._name, namespace=self._namespace) + if table_meta: + table_meta.destroy_metas() + return True + return False + def check_address(self): return True @@ -128,10 +130,6 @@ def put_all(self, kv_list: Iterable, **kwargs): # self._update_write_access_time() self._put_all(kv_list, **kwargs) - def put_meta(self, kv_list: Iterable, **kwargs): - # self._update_write_access_time() - self._put_meta(kv_list, **kwargs) - def collect(self, **kwargs) -> list: # self._update_read_access_time() return self._collect(**kwargs) @@ -150,26 +148,10 @@ def destroy(self): self.meta.destroy_metas() self._destroy() - def save_as(self, address, name, namespace, partitions=None, **kwargs): - table = self._save_as(address, name, namespace, partitions, **kwargs) - table.create_meta(**kwargs) - return table - - def _update_read_access_time(self, read_access_time=None): - read_access_time = current_timestamp() if not read_access_time else read_access_time - self._meta.update_metas(read_access_time=read_access_time) - - def _update_write_access_time(self, write_access_time=None): - write_access_time = current_timestamp() if not write_access_time else write_access_time - self._meta.update_metas(write_access_time=write_access_time) - # to be implemented def _put_all(self, kv_list: Iterable, **kwargs): raise NotImplementedError() - def _put_meta(self, kv_list: Iterable, **kwargs): - raise NotImplementedError() - def _collect(self, **kwargs) -> list: raise NotImplementedError() @@ -196,16 +178,15 @@ def __init__(self, name, namespace, new=False, create_address=True): self.store_type = None self.options = None self.partitions = None - self.in_serialized = None self.have_head = None - self.delimiter = None self.extend_sid = False self.auto_increasing_sid = None - self.schema = None + self.data_meta = None + self.data_type = None self.count = None self.part_of_data = None self.description = None - self.origin = None + self.source = None self.disable = None self.create_time = None self.update_time = None @@ -213,8 +194,8 @@ def __init__(self, name, namespace, new=False, create_address=True): self.write_access_time = None if self.options is None: self.options = {} - if self.schema is None: - self.schema = {} + if self.data_meta is None: + self.data_meta = {} if self.part_of_data is None: self.part_of_data = [] if not new: @@ -250,8 +231,9 @@ def exists(self): def create(self): table_meta = StorageTableMetaModel() table_meta.f_create_time = current_timestamp() - table_meta.f_schema = {} + table_meta.f_data_meta = {} table_meta.f_part_of_data = [] + table_meta.f_source = {} for k, v in self.to_dict().items(): attr_name = 'f_%s' % k if hasattr(StorageTableMetaModel, attr_name): @@ -261,13 +243,14 @@ def create(self): if rows != 1: raise Exception("create table meta failed") except peewee.IntegrityError as e: - if e.args[0] == 1062: - # warning - pass - elif isinstance(e.args[0], str) and "UNIQUE constraint failed" in e.args[0]: - pass - else: - raise e + # if e.args[0] == 1062: + # # warning + # pass + # elif isinstance(e.args[0], str) and "UNIQUE constraint failed" in e.args[0]: + # pass + # else: + # raise e + pass except Exception as e: raise e @@ -301,7 +284,7 @@ def query_table_meta(cls, filter_fields, query_fields=None): return [] @DB.connection_context() - def update_metas(self, schema=None, count=None, part_of_data=None, description=None, partitions=None, + def update_metas(self, data_meta=None, count=None, part_of_data=None, description=None, partitions=None, in_serialized=None, **kwargs): meta_info = {} for k, v in locals().items(): @@ -377,11 +360,8 @@ def get_options(self): def get_partitions(self): return self.partitions - def get_in_serialized(self): - return self.in_serialized - def get_id_delimiter(self): - return self.delimiter + return self.data_meta.get("delimiter", ",") def get_extend_sid(self): return self.extend_sid @@ -392,14 +372,14 @@ def get_auto_increasing_sid(self): def get_have_head(self): return self.have_head - def get_origin(self): - return self.origin + def get_source(self): + return self.source def get_disable(self): return self.disable - def get_schema(self): - return self.schema + def get_data_meta(self): + return self.data_meta def get_count(self): return self.count diff --git a/python/fate_flow/engine/storage/_types.py b/python/fate_flow/engine/storage/_types.py index 9b00624fb..366f0d33f 100644 --- a/python/fate_flow/engine/storage/_types.py +++ b/python/fate_flow/engine/storage/_types.py @@ -16,7 +16,13 @@ DEFAULT_ID_DELIMITER = "," -class StorageTableOrigin(object): +class DataType: + TABLE = "table" + DATAFRAME = "dataframe" + FILE = "file" + + +class StorageOrigin(object): TABLE_BIND = "table_bind" READER = "reader" UPLOAD = "upload" @@ -30,9 +36,11 @@ class StorageEngine(object): MYSQL = 'mysql' SIMPLE = 'simple' PATH = 'path' + FILE = 'file' HIVE = 'hive' - LOCALFS = 'localfs' API = 'api' + HTTP = 'http' + HTTPS = 'https' class StandaloneStoreType(object): diff --git a/python/fate_flow/engine/storage/eggroll/_session.py b/python/fate_flow/engine/storage/eggroll/_session.py index 6f5290714..333576d67 100644 --- a/python/fate_flow/engine/storage/eggroll/_session.py +++ b/python/fate_flow/engine/storage/eggroll/_session.py @@ -18,7 +18,7 @@ from eggroll.roll_pair.roll_pair import RollPairContext from fate_flow.engine.storage import EggRollStoreType, StorageEngine, StorageSessionBase from fate_flow.engine.storage.eggroll import StorageTable -from fate_flow.entity.address_types import AddressABC, EggRollAddress +from fate_flow.entity.types import AddressABC, EggRollAddress class StorageSession(StorageSessionBase): diff --git a/python/fate_flow/engine/storage/eggroll/_table.py b/python/fate_flow/engine/storage/eggroll/_table.py index a962aaa04..fbe420704 100644 --- a/python/fate_flow/engine/storage/eggroll/_table.py +++ b/python/fate_flow/engine/storage/eggroll/_table.py @@ -35,19 +35,17 @@ def __init__( address=address, partitions=partitions, options=options, - engine=StorageEngine.EGGROLL, - store_type=store_type, + engine=StorageEngine.EGGROLL ) + self._store_type = store_type self._context = context self._options["store_type"] = self._store_type self._options["total_partitions"] = partitions self._options["create_if_missing"] = True - self._table = self._context.load(namespace=self.namespace, name=self.name, options=self._options) - self._meta_table = self._context.load(namespace=self.namespace, name=self.meta_name, options=self._options) + self._table = self._context.load(namespace=self.address.namespace, name=self.address.name, options=self._options) def _save_as(self, address, name, namespace, partitions=None, **kwargs): - self._table.save_as(name=name, namespace=namespace) - + self._table.save_as(name=address.name, namespace=address.namespace) table = StorageTable( context=self._context, address=address, @@ -60,16 +58,11 @@ def _save_as(self, address, name, namespace, partitions=None, **kwargs): def _put_all(self, kv_list: Iterable, **kwargs): return self._table.put_all(kv_list) - def _put_meta(self, kv_list: Iterable, **kwargs): - return self._meta_table.put_all(kv_list) - - def _collect(self, **kwargs) -> list: return self._table.get_all(**kwargs) def _destroy(self): self._table.destroy() - self._meta_table.destory() def _count(self, **kwargs): return self._table.count() diff --git a/python/fate_flow/engine/storage/file/__init__.py b/python/fate_flow/engine/storage/file/__init__.py new file mode 100644 index 000000000..786c7bcca --- /dev/null +++ b/python/fate_flow/engine/storage/file/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage.file._session import StorageSession +from fate_flow.engine.storage.file._table import StorageTable + +__all__ = ["StorageTable", "StorageSession"] diff --git a/python/fate_flow/engine/storage/file/_session.py b/python/fate_flow/engine/storage/file/_session.py new file mode 100644 index 000000000..8adb53a4a --- /dev/null +++ b/python/fate_flow/engine/storage/file/_session.py @@ -0,0 +1,38 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage import StorageSessionBase, StorageEngine +from fate_flow.engine.storage.file._table import StorageTable +from fate_flow.entity.types import AddressABC, FileAddress + + +class StorageSession(StorageSessionBase): + def __init__(self, session_id, options=None): + super(StorageSession, self).__init__(session_id=session_id, engine=StorageEngine.FILE) + + def table(self, address: AddressABC, name, namespace, partitions, storage_type=None, options=None, **kwargs): + if isinstance(address, FileAddress): + return StorageTable(address=address, name=name, namespace=namespace, + partitions=partitions) + raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") + + def cleanup(self, name, namespace): + pass + + def stop(self): + pass + + def kill(self): + pass diff --git a/python/fate_flow/engine/storage/file/_table.py b/python/fate_flow/engine/storage/file/_table.py new file mode 100644 index 000000000..d88da3ce6 --- /dev/null +++ b/python/fate_flow/engine/storage/file/_table.py @@ -0,0 +1,132 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import os +from typing import Iterable + +from pyarrow import fs + +from fate_flow.engine.storage import StorageTableBase, StorageEngine +from fate_flow.manager.data.data_manager import DataManager +from fate_flow.utils.log import getLogger + +LOGGER = getLogger() + + +class StorageTable(StorageTableBase): + def __init__( + self, + address=None, + name: str = None, + namespace: str = None, + partitions: int = 1, + options=None, + ): + super(StorageTable, self).__init__( + name=name, + namespace=namespace, + address=address, + partitions=partitions, + options=options, + engine=StorageEngine.FILE + ) + self._local_fs_client = fs.LocalFileSystem() + + @property + def path(self): + return self._address.path + + def _put_all( + self, kv_list: Iterable, append=True, assume_file_exist=False, **kwargs + ): + LOGGER.info(f"put in file: {self.path}") + + self._local_fs_client.create_dir(os.path.dirname(self.path)) + + if append and (assume_file_exist or self._exist()): + stream = self._local_fs_client.open_append_stream( + path=self.path, compression=None + ) + else: + stream = self._local_fs_client.open_output_stream( + path=self.path, compression=None + ) + + counter = self._meta.get_count() if self._meta.get_count() else 0 + with io.TextIOWrapper(stream) as writer: + for k, v in kv_list: + writer.write(DataManager.serialize_data(k, v)) + writer.write("\n") + counter = counter + 1 + self._meta.update_metas(count=counter) + + def _collect(self, **kwargs) -> list: + for line in self._as_generator(): + yield DataManager.deserialize_data(line.rstrip()) + + def _read(self) -> list: + for line in self._as_generator(): + yield line + + def _destroy(self): + # use try/catch to avoid stop while deleting an non-exist file + try: + self._local_fs_client.delete_file(self.path) + except Exception as e: + LOGGER.debug(e) + + def _count(self): + count = 0 + for _ in self._as_generator(): + count += 1 + return count + + def close(self): + pass + + def _exist(self): + info = self._local_fs_client.get_file_info([self.path])[0] + return info.type != fs.FileType.NotFound + + def _as_generator(self): + info = self._local_fs_client.get_file_info([self.path])[0] + if info.type == fs.FileType.NotFound: + raise FileNotFoundError(f"file {self.path} not found") + + elif info.type == fs.FileType.File: + with io.TextIOWrapper( + buffer=self._local_fs_client.open_input_stream(self.path), encoding="utf-8" + ) as reader: + for line in reader: + yield line + else: + selector = fs.FileSelector(self.path) + file_infos = self._local_fs_client.get_file_info(selector) + for file_info in file_infos: + if file_info.base_name.startswith(".") or file_info.base_name.startswith("_"): + continue + assert ( + file_info.is_file + ), f"{self.path} is directory contains a subdirectory: {file_info.path}" + with io.TextIOWrapper( + buffer=self._local_fs_client.open_input_stream( + f"{self._address.file_path:}/{file_info.path}" + ), + encoding="utf-8", + ) as reader: + for line in reader: + yield line diff --git a/python/fate_flow/engine/storage/hdfs/__init__.py b/python/fate_flow/engine/storage/hdfs/__init__.py new file mode 100644 index 000000000..31c1dbd35 --- /dev/null +++ b/python/fate_flow/engine/storage/hdfs/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage.hdfs._table import StorageTable +from fate_flow.engine.storage.hdfs._session import StorageSession + +__all__ = ["StorageTable", "StorageSession"] diff --git a/python/fate_flow/engine/storage/hdfs/_session.py b/python/fate_flow/engine/storage/hdfs/_session.py new file mode 100644 index 000000000..199798e55 --- /dev/null +++ b/python/fate_flow/engine/storage/hdfs/_session.py @@ -0,0 +1,43 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage import StorageSessionBase, StorageEngine +from fate_flow.engine.storage.hdfs._table import StorageTable +from fate_flow.entity.types import AddressABC, HDFSAddress + + +class StorageSession(StorageSessionBase): + def __init__(self, session_id, options=None): + super(StorageSession, self).__init__(session_id=session_id, engine=StorageEngine.HDFS) + + def table(self, address: AddressABC, name, namespace, partitions, store_type=None, options=None, **kwargs): + if isinstance(address, HDFSAddress): + return StorageTable( + address=address, + name=name, + namespace=namespace, + partitions=partitions, + options=options + ) + raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") + + def cleanup(self, name, namespace): + pass + + def stop(self): + pass + + def kill(self): + pass diff --git a/python/fate_flow/engine/storage/hdfs/_table.py b/python/fate_flow/engine/storage/hdfs/_table.py new file mode 100644 index 000000000..257ceccd9 --- /dev/null +++ b/python/fate_flow/engine/storage/hdfs/_table.py @@ -0,0 +1,145 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +from typing import Iterable + +from pyarrow import fs + +from fate_flow.engine.storage import StorageTableBase +from fate_flow.engine.storage._types import StorageEngine +from fate_flow.manager.data.data_manager import DataManager +from fate_flow.utils.log import getLogger + + +LOGGER = getLogger() + + +class StorageTable(StorageTableBase): + def __init__( + self, + address=None, + name: str = None, + namespace: str = None, + partitions: int = 1, + options=None, + ): + super(StorageTable, self).__init__( + name=name, + namespace=namespace, + address=address, + partitions=partitions, + options=options, + engine=StorageEngine.HDFS + ) + try: + from pyarrow import HadoopFileSystem + HadoopFileSystem(self.path) + except Exception as e: + LOGGER.warning(f"load libhdfs failed: {e}") + + self._hdfs_client = fs.HadoopFileSystem.from_uri(self.path) + + def check_address(self): + return self._exist() + + def _put_all( + self, kv_list: Iterable, append=True, assume_file_exist=False, **kwargs + ): + + client = self._hdfs_client + path = self.file_path + LOGGER.info(f"put in hdfs file: {path}") + if append and (assume_file_exist or self._exist(path)): + stream = client.open_append_stream( + path=path, compression=None + ) + else: + stream = client.open_output_stream( + path=path, compression=None + ) + + counter = self._meta.get_count() if self._meta.get_count() else 0 + with io.TextIOWrapper(stream) as writer: + for k, v in kv_list: + writer.write(DataManager.serialize_data(k, v)) + writer.write("\n") + counter = counter + 1 + self._meta.update_metas(count=counter) + + def _collect(self, **kwargs) -> list: + for line in self._as_generator(): + yield DataManager.deserialize_data(line.rstrip()) + + def _read(self) -> list: + for line in self._as_generator(): + yield line + + def _destroy(self): + self._hdfs_client.delete_file(self.file_path) + + def _count(self): + count = 0 + if self._meta.get_count(): + return self._meta.get_count() + for _ in self._as_generator(): + count += 1 + return count + + def close(self): + pass + + @property + def path(self) -> str: + return f"{self._address.name_node}/{self._address.path}" + + @property + def file_path(self) -> str: + return f"{self._address.path}" + + def _exist(self, path=None): + if not path: + path = self.file_path + info = self._hdfs_client.get_file_info([path])[0] + return info.type != fs.FileType.NotFound + + def _as_generator(self): + file = self.file_path + LOGGER.info(f"as generator: {file}") + info = self._hdfs_client.get_file_info([file])[0] + if info.type == fs.FileType.NotFound: + raise FileNotFoundError(f"file {file} not found") + + elif info.type == fs.FileType.File: + with io.TextIOWrapper( + buffer=self._hdfs_client.open_input_stream(self.path), encoding="utf-8" + ) as reader: + for line in reader: + yield line + else: + selector = fs.FileSelector(file) + file_infos = self._hdfs_client.get_file_info(selector) + for file_info in file_infos: + if file_info.base_name == "_SUCCESS": + continue + assert ( + file_info.is_file + ), f"{self.path} is directory contains a subdirectory: {file_info.path}" + with io.TextIOWrapper( + buffer=self._hdfs_client.open_input_stream(file_info.path), + encoding="utf-8", + ) as reader: + for line in reader: + yield line diff --git a/python/fate_flow/engine/storage/standalone/_session.py b/python/fate_flow/engine/storage/standalone/_session.py index 6e0107415..6afa1941b 100644 --- a/python/fate_flow/engine/storage/standalone/_session.py +++ b/python/fate_flow/engine/storage/standalone/_session.py @@ -15,7 +15,7 @@ # from fate_flow.engine.storage import StorageSessionBase, StorageEngine from fate_flow.engine.storage.standalone._standalone import Session -from fate_flow.entity.address_types import AddressABC, StandaloneAddress +from fate_flow.entity.types import AddressABC, StandaloneAddress class StorageSession(StorageSessionBase): diff --git a/python/fate_flow/engine/storage/standalone/_standalone.py b/python/fate_flow/engine/storage/standalone/_standalone.py index e3d05066c..1a4bc2344 100644 --- a/python/fate_flow/engine/storage/standalone/_standalone.py +++ b/python/fate_flow/engine/storage/standalone/_standalone.py @@ -22,7 +22,7 @@ import time import typing import uuid -from collections import Iterable +from collections.abc import Iterable from concurrent.futures import ProcessPoolExecutor as Executor from contextlib import ExitStack from functools import partial @@ -34,7 +34,7 @@ import lmdb import numpy as np -from fate_flow.utils import file_utils +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME from fate_flow.utils.log import getLogger LOGGER = getLogger("storage") @@ -690,7 +690,7 @@ def _put_to_meta_table(key, value): _get_meta_table().put(key, value) -_data_dir = Path(file_utils.get_project_base_directory()).joinpath("data").absolute() +_data_dir = Path(STANDALONE_DATA_HOME).absolute() def _get_data_dir(): diff --git a/python/fate_flow/engine/storage/standalone/_table.py b/python/fate_flow/engine/storage/standalone/_table.py index 6635ed497..8ba7c80f2 100644 --- a/python/fate_flow/engine/storage/standalone/_table.py +++ b/python/fate_flow/engine/storage/standalone/_table.py @@ -19,7 +19,6 @@ from fate_flow.engine.storage.standalone._standalone import Session - class StorageTable(StorageTableBase): def __init__( self, @@ -38,31 +37,20 @@ def __init__( partitions=partitions, options=options, engine=StorageEngine.STANDALONE, - store_type=store_type, ) + self._store_type = store_type self._session = session self._table = self._session.create_table( - namespace=self.namespace, - name=self.name, + namespace=self.address.namespace, + name=self.address.name, partitions=partitions, need_cleanup=self._store_type == StandaloneStoreType.ROLLPAIR_IN_MEMORY, error_if_exist=False, ) - self._meta_table = self._session.create_table( - namespace=self.namespace, - name=self.meta_name, - partitions=partitions, - need_cleanup=self._store_type == StandaloneStoreType.ROLLPAIR_IN_MEMORY, - error_if_exist=False, - ) - def _put_all(self, kv_list: Iterable, **kwargs): return self._table.put_all(kv_list) - def _put_meta(self, kv_list: Iterable, **kwargs): - return self._meta_table.put_all(kv_list) - def _collect(self, **kwargs): return self._table.collect(**kwargs) @@ -71,10 +59,9 @@ def _count(self): def _destroy(self): self._table.destroy() - self._meta_table.destroy() def _save_as(self, address, name, namespace, partitions=None, **kwargs): - self._table.save_as(name=name, namespace=namespace) + self._table.save_as(name=address.name, namespace=address.namespace) table = StorageTable( session=self._session, diff --git a/python/fate_flow/entity/__init__.py b/python/fate_flow/entity/__init__.py index 854b85271..1bb3927e4 100644 --- a/python/fate_flow/entity/__init__.py +++ b/python/fate_flow/entity/__init__.py @@ -13,4 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from ._base import BaseEntity, BaseModel + +from ._base import BaseEntity, BaseModel, CustomEnum + +__all__ = ["BaseEntity", "BaseModel", "CustomEnum"] diff --git a/python/fate_flow/entity/_base.py b/python/fate_flow/entity/_base.py index 8e47195a4..e13d2b4e6 100644 --- a/python/fate_flow/entity/_base.py +++ b/python/fate_flow/entity/_base.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from enum import Enum + from pydantic import BaseModel as Base from fate_flow.utils.base_utils import BaseType @@ -30,4 +32,22 @@ def to_dict(self): return d def __str__(self): - return str(self.to_dict()) \ No newline at end of file + return str(self.to_dict()) + + +class CustomEnum(Enum): + @classmethod + def valid(cls, value): + try: + cls(value) + return True + except: + return False + + @classmethod + def values(cls): + return [member.value for member in cls.__members__.values()] + + @classmethod + def names(cls): + return [member.name for member in cls.__members__.values()] \ No newline at end of file diff --git a/python/fate_flow/entity/address_types.py b/python/fate_flow/entity/address_types.py deleted file mode 100644 index cbc4d645f..000000000 --- a/python/fate_flow/entity/address_types.py +++ /dev/null @@ -1,200 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import abc - - -class AddressABC(metaclass=abc.ABCMeta): - ... - - -class AddressBase(AddressABC): - def __init__(self, connector_name=None): - pass - - @property - def connector(self): - return {} - - @property - def storage_engine(self): - return - - -class StandaloneAddress(AddressBase): - def __init__(self, home=None, name=None, namespace=None, storage_type=None, connector_name=None): - self.home = home - self.name = name - self.namespace = namespace - self.storage_type = storage_type - super(StandaloneAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return (self.home, self.name, self.namespace, self.storage_type).__hash__() - - def __str__(self): - return f"StandaloneAddress(name={self.name}, namespace={self.namespace})" - - def __repr__(self): - return self.__str__() - - @property - def connector(self): - return {"home": self.home} - - -class EggRollAddress(AddressBase): - def __init__(self, home=None, name=None, namespace=None, connector_name=None): - self.name = name - self.namespace = namespace - self.home = home - super(EggRollAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return (self.home, self.name, self.namespace).__hash__() - - def __str__(self): - return f"EggRollAddress(name={self.name}, namespace={self.namespace})" - - def __repr__(self): - return self.__str__() - - @property - def connector(self): - return {"home": self.home} - - -class HDFSAddress(AddressBase): - def __init__(self, name_node=None, path=None, connector_name=None): - self.name_node = name_node - self.path = path - super(HDFSAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return (self.name_node, self.path).__hash__() - - def __str__(self): - return f"HDFSAddress(name_node={self.name_node}, path={self.path})" - - def __repr__(self): - return self.__str__() - - @property - def connector(self): - return {"name_node": self.name_node} - - -class PathAddress(AddressBase): - def __init__(self, path=None, connector_name=None): - self.path = path - super(PathAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return self.path.__hash__() - - def __str__(self): - return f"PathAddress(path={self.path})" - - def __repr__(self): - return self.__str__() - - -class ApiAddress(AddressBase): - def __init__(self, method="POST", url=None, header=None, body=None, connector_name=None): - self.method = method - self.url = url - self.header = header if header else {} - self.body = body if body else {} - super(ApiAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return (self.method, self.url).__hash__() - - def __str__(self): - return f"ApiAddress(url={self.url})" - - def __repr__(self): - return self.__str__() - - -class MysqlAddress(AddressBase): - def __init__(self, user=None, passwd=None, host=None, port=None, db=None, name=None, connector_name=None): - self.user = user - self.passwd = passwd - self.host = host - self.port = port - self.db = db - self.name = name - self.connector_name = connector_name - super(MysqlAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return (self.host, self.port, self.db, self.name).__hash__() - - def __str__(self): - return f"MysqlAddress(db={self.db}, name={self.name})" - - def __repr__(self): - return self.__str__() - - @property - def connector(self): - return {"user": self.user, "passwd": self.passwd, "host": self.host, "port": self.port, "db": self.db} - - -class HiveAddress(AddressBase): - def __init__(self, host=None, name=None, port=10000, username=None, database='default', auth_mechanism='PLAIN', - password=None, connector_name=None): - self.host = host - self.username = username - self.port = port - self.database = database - self.auth_mechanism = auth_mechanism - self.password = password - self.name = name - super(HiveAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return (self.host, self.port, self.database, self.name).__hash__() - - def __str__(self): - return f"HiveAddress(database={self.database}, name={self.name})" - - def __repr__(self): - return self.__str__() - - @property - def connector(self): - return { - "host": self.host, - "port": self.port, - "username": self.username, - "password": self.password, - "auth_mechanism": self.auth_mechanism, - "database": self.database} - - -class LocalFSAddress(AddressBase): - def __init__(self, path=None, connector_name=None): - self.path = path - super(LocalFSAddress, self).__init__(connector_name=connector_name) - - def __hash__(self): - return (self.path).__hash__() - - def __str__(self): - return f"LocalFSAddress(path={self.path})" - - def __repr__(self): - return self.__str__() diff --git a/python/fate_flow/entity/code/__init__.py b/python/fate_flow/entity/code/__init__.py new file mode 100644 index 000000000..868f7c581 --- /dev/null +++ b/python/fate_flow/entity/code/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from ._schedule import * +from ._api import * +from ._process import * diff --git a/python/fate_flow/entity/code/_api.py b/python/fate_flow/entity/code/_api.py new file mode 100644 index 000000000..960285208 --- /dev/null +++ b/python/fate_flow/entity/code/_api.py @@ -0,0 +1,73 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class ReturnCode: + + class Base: + SUCCESS = 0 + + class Job: + PARAMS_ERROR = 1000 + NOT_FOUND = 1001 + CREATE_JOB_FAILED = 1002 + UPDATE_FAILED = 1003 + KILL_FAILED = 1004 + RESOURCE_EXCEPTION = 1005 + INHERITANCE_FAILED = 1006 + + class Task: + NOT_FOUND = 2000 + START_FAILED = 2001 + UPDATE_FAILED = 2002 + KILL_FAILED = 2003 + RESOURCE_EXCEPTION = 2004 + NO_FOUND_MODEL_OUTPUT = 2005 + TASK_RUN_FAILED = 2006 + COMPONENT_RUN_FAILED = 2007 + NO_FOUND_RUN_RESULT = 2008 + + class Site: + IS_STANDALONE = 3000 + + class Provider: + PARAMS_ERROR = 4000 + DEVICE_NOT_SUPPORTED = 4001 + + class API: + EXPIRED = 5000 + INVALID_PARAMETER = 5001 + NO_FOUND_APPID = 5002 + VERIFY_FAILED = 5003 + AUTHENTICATION_FAILED = 5004 + NO_PERMISSION = 5005 + PERMISSION_OPERATE_ERROR = 5006 + NO_FOUND_FILE = 5007 + COMPONENT_OUTPUT_EXCEPTION = 5008 + ROLE_TYPE_ERROR = 5009 + + class Server: + EXCEPTION = 6000 + FUNCTION_RESTRICTED = 6001 + RESPONSE_EXCEPTION = 6002 + NO_FOUND = 6003 + NO_FOUND_INSTANCE = 6004 + + class Table: + NO_FOUND = 7001 + EXISTS = 7002 + + class File: + FILE_NOT_FOUND = 8001 + FILE_EXISTS = 8002 diff --git a/python/fate_flow/entity/code/_process.py b/python/fate_flow/entity/code/_process.py new file mode 100644 index 000000000..15ace6ed0 --- /dev/null +++ b/python/fate_flow/entity/code/_process.py @@ -0,0 +1,24 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import IntEnum + +from fate_flow.entity import CustomEnum + + +class KillProcessRetCode(IntEnum, CustomEnum): + KILLED = 0 + NOT_FOUND = 1 + ERROR_PID = 2 diff --git a/python/fate_flow/entity/code/_schedule.py b/python/fate_flow/entity/code/_schedule.py new file mode 100644 index 000000000..38300bd4c --- /dev/null +++ b/python/fate_flow/entity/code/_schedule.py @@ -0,0 +1,27 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class SchedulingStatusCode(object): + SUCCESS = 0 + NO_RESOURCE = 1 + PASS = 1 + NO_NEXT = 2 + HAVE_NEXT = 3 + FAILED = 4 + + +class FederatedSchedulingStatusCode(object): + SUCCESS = 0 + FAILED = 1 diff --git a/python/fate_flow/entity/dag_structures.py b/python/fate_flow/entity/dag_structures.py deleted file mode 100644 index 6256bc26b..000000000 --- a/python/fate_flow/entity/dag_structures.py +++ /dev/null @@ -1,218 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Optional, Dict, List, Union, Any, Literal, TypeVar -from pydantic import BaseModel - -# task -class IOArtifact(BaseModel): - name: str - uri: str - metadata: Optional[dict] - - -class InputSpec(BaseModel): - parameters: Optional[Dict[str, Any]] - artifacts: Optional[IOArtifact] - - -class TaskRuntimeInputSpec(BaseModel): - parameters: Optional[Dict[str, Any]] - artifacts: Optional[Dict[str, IOArtifact]] - - -class TaskRuntimeOutputSpec(BaseModel): - artifacts: Dict[str, IOArtifact] - - -class MLMDSpec(BaseModel): - type: str - metadata: Dict[str, Any] - - -class LOGGERSpec(BaseModel): - type: str - metadata: Dict[str, Any] - - -class ComputingBackendSpec(BaseModel): - type: str - metadata: Dict[str, Any] - - -class FederationBackendSpec(BaseModel): - type: str - metadata: Dict[str, Any] - - -class OutputModelSpec(BaseModel): - type: str - metadata: Dict[str, str] - - -class OutputMetricSpec(BaseModel): - type: str - metadata: Dict[str, str] - - -class OutputDataSpec(BaseModel): - type: str - metadata: Dict[str, str] - - -class OutputSpec(BaseModel): - model: OutputModelSpec - metric: OutputMetricSpec - data: OutputDataSpec - - -class RuntimeConfSpec(BaseModel): - output: OutputSpec - mlmd: MLMDSpec - logger: LOGGERSpec - device: Dict[str, str] - computing: ComputingBackendSpec - federation: FederationBackendSpec - - -class TaskScheduleSpec(BaseModel): - task_id: Optional[str] - party_task_id: Optional[str] - component: Optional[str] - role: Optional[str] - stage: Optional[str] - party_id: Optional[str] - inputs: Optional[TaskRuntimeInputSpec] - conf: RuntimeConfSpec - -# component -class ParameterSpec(BaseModel): - type: str - default: Any - optional: bool - - -class ArtifactSpec(BaseModel): - type: str - optional: bool - stages: Optional[List[str]] - roles: Optional[List[str]] - - -class InputDefinitionsSpec(BaseModel): - parameters: Dict[str, ParameterSpec] - artifacts: Dict[str, ArtifactSpec] - - -class OutputDefinitionsSpec(BaseModel): - artifacts: Dict[str, ArtifactSpec] - - -class ComponentSpec(BaseModel): - name: str - description: str - provider: str - version: str - labels: List[str] = ["trainable"] - roles: List[str] - input_definitions: InputDefinitionsSpec - output_definitions: OutputDefinitionsSpec - - -class RuntimeOutputChannelSpec(BaseModel): - producer_task: str - output_artifact_key: str - - -class RuntimeInputDefinition(BaseModel): - parameters: Optional[Dict[str, Any]] - artifacts: Optional[Dict[str, Dict[str, RuntimeOutputChannelSpec]]] - -# dag -class PartySpec(BaseModel): - role: Union[Literal["guest", "host", "arbiter"]] - party_id: List[str] - - -class RuntimeTaskOutputChannelSpec(BaseModel): - producer_task: str - output_artifact_key: str - roles: Optional[List[Literal["guest", "host", "arbiter"]]] - - -class ModelWarehouseChannelSpec(BaseModel): - model_id: Optional[str] - model_version: Optional[Union[str, int]] - producer_task: str - output_artifact_key: str - roles: Optional[List[Literal["guest", "host", "arbiter"]]] - - -InputChannelSpec = TypeVar("InputChannelSpec", RuntimeTaskOutputChannelSpec, ModelWarehouseChannelSpec) - - -class TaskRuntimeInputDefinition(BaseModel): - parameters: Optional[Dict[str, Any]] - artifacts: Optional[Dict[str, Dict[str, Union[InputChannelSpec, List[InputChannelSpec]]]]] - - -class TaskSpec(BaseModel): - component_ref: str - dependent_tasks: Optional[List[str]] - inputs: Optional[TaskRuntimeInputDefinition] - parties: Optional[List[PartySpec]] - conf: Optional[Dict[Any, Any]] - stage: Optional[Union[Literal["train", "predict", "default"]]] - - -class PartyTaskRefSpec(BaseModel): - inputs: TaskRuntimeInputDefinition - conf: Optional[Dict] - - -class PartyTaskSpec(BaseModel): - parties: Optional[List[PartySpec]] - tasks: Dict[str, PartyTaskRefSpec] - conf: Optional[dict] - - -class TaskConfSpec(BaseModel): - task_cores: int - engine: Dict[str, Any] - - -class JobConfSpec(BaseModel): - scheduler_party_id: Optional[str] - initiator_party_id: Optional[str] - inherit: Optional[Dict[str, Any]] - task_parallelism: Optional[int] - task_cores: Optional[int] - federated_status_collect_type: Optional[str] - auto_retries: Optional[int] - model_id: Optional[str] - model_version: Optional[Union[str, int]] - task: Optional[TaskConfSpec] - - -class DAGSpec(BaseModel): - parties: List[PartySpec] - conf: Optional[JobConfSpec] - stage: Optional[Union[Literal["train", "predict", "default"]]] - tasks: Dict[str, TaskSpec] - party_tasks: Optional[Dict[str, PartyTaskSpec]] - - -class DAGSchema(BaseModel): - dag: DAGSpec - schema_version: str diff --git a/python/fate_flow/entity/spec/__init__.py b/python/fate_flow/entity/spec/__init__.py new file mode 100644 index 000000000..878d3a9c5 --- /dev/null +++ b/python/fate_flow/entity/spec/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/fate_flow/entity/spec/dag/__init__.py b/python/fate_flow/entity/spec/dag/__init__.py new file mode 100644 index 000000000..b8c5585c6 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/__init__.py @@ -0,0 +1,32 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.spec.dag._output import ComponentOutputMeta, MetricData +from fate_flow.entity.spec.dag._party import PartySpec +from fate_flow.entity.spec.dag._job import DAGSchema, DAGSpec, JobConfSpec, TaskConfSpec, TaskSpec, PartyTaskSpec, \ + InheritConfSpec, PartyTaskRefSpec +from fate_flow.entity.spec.dag._task import TaskConfigSpec, PreTaskConfigSpec, TaskRuntimeConfSpec, \ + TaskCleanupConfigSpec +from fate_flow.entity.spec.dag._artifact import RuntimeTaskOutputChannelSpec, DataWarehouseChannelSpec, \ + ModelWarehouseChannelSpec, SourceInputArtifactSpec, RuntimeInputArtifacts, FlowRuntimeInputArtifacts,\ + ArtifactInputApplySpec, Metadata, RuntimeTaskOutputChannelSpec, \ + ArtifactOutputApplySpec, ModelWarehouseChannelSpec, ArtifactOutputSpec, ArtifactSource +from fate_flow.entity.spec.dag._component import ComponentSpec, ComponentIOArtifactsTypeSpec +from fate_flow.entity.spec.dag._computing import EggrollComputingSpec, SparkComputingSpec, StandaloneComputingSpec +from fate_flow.entity.spec.dag._federation import StandaloneFederationSpec, RollSiteFederationSpec, OSXFederationSpec, \ + PulsarFederationSpec, RabbitMQFederationSpec +from fate_flow.entity.spec.dag._logger import FlowLogger +from fate_flow.entity.spec.dag._mlmd import MLMDSpec +from fate_flow.entity.spec.dag._device import LauncherSpec diff --git a/python/fate_flow/entity/spec/dag/_artifact.py b/python/fate_flow/entity/spec/dag/_artifact.py new file mode 100644 index 000000000..2b862eb6f --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_artifact.py @@ -0,0 +1,182 @@ +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from typing import Optional, List, Literal, TypeVar, Dict, Union + +import pydantic + +# see https://www.rfc-editor.org/rfc/rfc3986#appendix-B +# scheme = $2 +# authority = $4 +# path = $5 +# query = $7 +# fragment = $9 +_uri_regex = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + + +class ArtifactSource(pydantic.BaseModel): + task_id: str + party_task_id: str + task_name: str + component: str + output_artifact_key: str + output_index: Optional[int] = None + + +class Metadata(pydantic.BaseModel): + class DataOverview(pydantic.BaseModel): + count: Optional[int] = None + samples: Optional[List] = None + metadata: dict = pydantic.Field(default_factory=dict) + name: Optional[str] = None + namespace: Optional[str] = None + model_overview: Optional[dict] = {} + data_overview: Optional[DataOverview] + source: Optional[ArtifactSource] = None + model_key: Optional[str] + type_name: Optional[str] + index: Optional[Union[int, None]] = None + + class Config: + extra = "forbid" + + +class ArtifactInputApplySpec(pydantic.BaseModel): + uri: str + metadata: Metadata + type_name: Optional[str] = None + + def get_uri(self) -> "URI": + return URI.from_string(self.uri) + + +class ArtifactOutputApplySpec(pydantic.BaseModel): + uri: str + _is_template: Optional[bool] = None + type_name: Optional[str] = None + + def get_uri(self, index) -> "URI": + if self.is_template(): + return URI.from_string(self.uri.format(index=index)) + else: + if index != 0: + raise ValueError(f"index should be 0, but got {index}") + return URI.from_string(self.uri) + + def is_template(self) -> bool: + return "{index}" in self.uri + + def _check_is_template(self) -> bool: + return "{index}" in self.uri + + @pydantic.validator("uri") + def _check_uri(cls, v, values) -> str: + if not _uri_regex.match(v): + raise pydantic.ValidationError(f"`{v}` is not valid uri") + return v + + +class ArtifactOutputSpec(pydantic.BaseModel): + uri: str + metadata: Metadata + type_name: str + consumed: Optional[bool] = None + + +class URI: + def __init__( + self, + schema: str, + path: str, + query: Optional[str] = None, + fragment: Optional[str] = None, + authority: Optional[str] = None, + ): + self.schema = schema + self.path = path + self.query = query + self.fragment = fragment + self.authority = authority + + @classmethod + def from_string(cls, uri: str) -> "URI": + match = _uri_regex.fullmatch(uri) + if match is None: + raise ValueError(f"`{uri}` is not valid uri") + _, schema, _, authority, path, _, query, _, fragment = match.groups() + return URI(schema=schema, path=path, query=query, fragment=fragment, authority=authority) + + @classmethod + def load_uri(cls, engine, address): + pass + + +class RuntimeTaskOutputChannelSpec(pydantic.BaseModel): + producer_task: str + output_artifact_key: str + roles: Optional[List[Literal["guest", "host", "arbiter", "local"]]] + + class Config: + extra = "forbid" + + +class DataWarehouseChannelSpec(pydantic.BaseModel): + job_id: Optional[str] + producer_task: Optional[str] + output_artifact_key: Optional[str] + roles: Optional[List[Literal["guest", "host", "arbiter", "local"]]] + namespace: Optional[str] + name: Optional[str] + + class Config: + extra = "forbid" + + +class ModelWarehouseChannelSpec(pydantic.BaseModel): + model_id: Optional[str] + model_version: Optional[str] + producer_task: str + output_artifact_key: str + roles: Optional[List[Literal["guest", "host", "arbiter", "local"]]] + + class Config: + extra = "forbid" + + +InputArtifactSpec = TypeVar("InputArtifactSpec", + RuntimeTaskOutputChannelSpec, + ModelWarehouseChannelSpec, + DataWarehouseChannelSpec) + + +SourceInputArtifactSpec = TypeVar("SourceInputArtifactSpec", + ModelWarehouseChannelSpec, + DataWarehouseChannelSpec) + + +class RuntimeInputArtifacts(pydantic.BaseModel): + data: Optional[Dict[str, Dict[str, Union[List[InputArtifactSpec], InputArtifactSpec]]]] + model: Optional[Dict[str, Dict[str, Union[List[InputArtifactSpec], InputArtifactSpec]]]] + + +class SourceInputArtifacts(pydantic.BaseModel): + data: Optional[Dict[str, Dict[str, Union[SourceInputArtifactSpec, List[SourceInputArtifactSpec]]]]] + model: Optional[Dict[str, Dict[str, Union[SourceInputArtifactSpec, List[SourceInputArtifactSpec]]]]] + + +class FlowRuntimeInputArtifacts(pydantic.BaseModel): + data: Optional[Dict[str, Union[InputArtifactSpec, List[InputArtifactSpec]]]] + model: Optional[Dict[str, Union[InputArtifactSpec, List[InputArtifactSpec]]]] diff --git a/python/fate_flow/entity/spec/dag/_component.py b/python/fate_flow/entity/spec/dag/_component.py new file mode 100644 index 000000000..fd64ed0bf --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_component.py @@ -0,0 +1,95 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Dict, List, Union, Any, Literal +from pydantic import BaseModel + + +class ParameterSpec(BaseModel): + type: str + default: Any + optional: bool + description: str = "" + type_meta: dict = {} + + +class ArtifactSpec(BaseModel): + type: str + optional: bool + stages: Optional[List[str]] + roles: Optional[List[str]] + description: str = "" + is_multi: bool + + +class InputArtifactsSpec(BaseModel): + data: Dict[str, ArtifactSpec] + model: Dict[str, ArtifactSpec] + + +class OutputArtifactsSpec(BaseModel): + data: Dict[str, ArtifactSpec] + model: Dict[str, ArtifactSpec] + metric: Dict[str, ArtifactSpec] + + +class ComponentSpec(BaseModel): + name: str + description: str + provider: str + version: str + labels: List[str] = ["trainable"] + roles: List[str] + parameters: Dict[str, ParameterSpec] + input_artifacts: InputArtifactsSpec + output_artifacts: OutputArtifactsSpec + + +class RuntimeOutputChannelSpec(BaseModel): + producer_task: str + output_artifact_key: str + + +class RuntimeInputDefinition(BaseModel): + parameters: Optional[Dict[str, Any]] + artifacts: Optional[Dict[str, Dict[str, RuntimeOutputChannelSpec]]] + + +class ArtifactTypeSpec(BaseModel): + type_name: str + uri_types: List[str] + path_type: Literal["file", "directory", "distributed"] + + +class ComponentIOArtifactTypeSpec(BaseModel): + name: str + is_multi: bool + optional: bool + types: List[ArtifactTypeSpec] + + +class ComponentIOInputsArtifactsTypeSpec(BaseModel): + data: List[ComponentIOArtifactTypeSpec] + model: List[ComponentIOArtifactTypeSpec] + + +class ComponentIOOutputsArtifactsTypeSpec(BaseModel): + data: List[ComponentIOArtifactTypeSpec] + model: List[ComponentIOArtifactTypeSpec] + metric: List[ComponentIOArtifactTypeSpec] + + +class ComponentIOArtifactsTypeSpec(BaseModel): + inputs: ComponentIOInputsArtifactsTypeSpec + outputs: ComponentIOOutputsArtifactsTypeSpec diff --git a/python/fate_flow/entity/spec/dag/_computing.py b/python/fate_flow/entity/spec/dag/_computing.py new file mode 100644 index 000000000..d2cc2736e --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_computing.py @@ -0,0 +1,48 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Literal, TypeVar + +import pydantic + + +class StandaloneComputingSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + computing_id: str + options: dict = {} + + type: Literal["standalone"] + metadata: MetadataSpec + + +class EggrollComputingSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + computing_id: str + options: dict = {} + + type: Literal["eggroll"] + metadata: MetadataSpec + + +class SparkComputingSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + computing_id: str + + type: Literal["spark"] + metadata: MetadataSpec + + +class CustomComputingSpec(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict diff --git a/python/fate_flow/entity/spec/dag/_device.py b/python/fate_flow/entity/spec/dag/_device.py new file mode 100644 index 000000000..ec3677094 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_device.py @@ -0,0 +1,33 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Literal + +import pydantic +from pydantic import typing + + +class CPUSpec(pydantic.BaseModel): + type: Literal["CPU"] + metadata: dict = {} + + +class GPUSpec(pydantic.BaseModel): + type: Literal["GPU"] + metadata: dict = {} + + +class LauncherSpec(pydantic.BaseModel): + name: str = "default" + conf: dict = {} diff --git a/python/fate_flow/hub/parser/default/_federation.py b/python/fate_flow/entity/spec/dag/_federation.py similarity index 98% rename from python/fate_flow/hub/parser/default/_federation.py rename to python/fate_flow/entity/spec/dag/_federation.py index 6d0b49a3a..0eee05a15 100644 --- a/python/fate_flow/hub/parser/default/_federation.py +++ b/python/fate_flow/entity/spec/dag/_federation.py @@ -12,14 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import ipaddress from typing import Dict, List, Literal, Optional import pydantic class PartySpec(pydantic.BaseModel): - role: Literal["guest", "host", "arbiter"] + role: Literal["guest", "host", "arbiter", "local"] partyid: str def tuple(self): diff --git a/python/fate_flow/entity/spec/dag/_job.py b/python/fate_flow/entity/spec/dag/_job.py new file mode 100644 index 000000000..2f522ac37 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_job.py @@ -0,0 +1,91 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional, Union, Literal, Dict, List, Any + +from pydantic import BaseModel + +from fate_flow.entity.spec.dag._party import PartySpec +from fate_flow.entity.spec.dag._artifact import RuntimeInputArtifacts, SourceInputArtifacts + + +class TaskSpec(BaseModel): + component_ref: str + dependent_tasks: Optional[List[str]] + parameters: Optional[Dict[Any, Any]] + inputs: Optional[RuntimeInputArtifacts] + parties: Optional[List[PartySpec]] + conf: Optional[Dict[Any, Any]] + stage: Optional[Union[Literal["train", "predict", "default", "cross_validation"]]] + + +class PartyTaskRefSpec(BaseModel): + parameters: Optional[Dict[Any, Any]] + inputs: Optional[SourceInputArtifacts] + conf: Optional[Dict] + + +class PartyTaskSpec(BaseModel): + parties: Optional[List[PartySpec]] + tasks: Optional[Dict[str, PartyTaskRefSpec]] = {} + conf: Optional[dict] + + +class EngineRunSpec(BaseModel): + name: str + conf: Optional[Dict] + + +class TaskConfSpec(BaseModel): + run: Optional[Dict] + provider: Optional[str] + + +class InheritConfSpec(BaseModel): + job_id: str + task_list: List[str] + + +class JobConfSpec(BaseModel): + class PipelineModel(BaseModel): + model_id: str + model_version: Union[str, int] + priority: Optional[int] + scheduler_party_id: Optional[str] + initiator_party_id: Optional[str] + inheritance: Optional[InheritConfSpec] + cores: Optional[int] + task_cores: Optional[int] + computing_partitions: Optional[int] + sync_type: Optional[Union[Literal["poll", "callback"]]] + auto_retries: Optional[int] + model_id: Optional[str] + model_version: Optional[Union[str, int]] + model_warehouse: Optional[PipelineModel] + task: Optional[TaskConfSpec] + engine: Optional[EngineRunSpec] + + +class DAGSpec(BaseModel): + parties: List[PartySpec] + conf: Optional[JobConfSpec] + stage: Optional[Union[Literal["train", "predict", "default", "cross_validation"]]] + tasks: Dict[str, TaskSpec] + party_tasks: Optional[Dict[str, PartyTaskSpec]] + + +class DAGSchema(BaseModel): + dag: DAGSpec + schema_version: str diff --git a/python/fate_flow/entity/spec/dag/_logger.py b/python/fate_flow/entity/spec/dag/_logger.py new file mode 100644 index 000000000..420e8f6b0 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_logger.py @@ -0,0 +1,156 @@ +import logging +import logging.config +import os + +import pydantic +from typing import Optional + +_LOGGER_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR"] + + +class FlowLogger(pydantic.BaseModel): + config: dict + + def install(self): + for _name, _conf in self.config.get("handlers", {}).items(): + if _conf.get("filename"): + os.makedirs(os.path.dirname(_conf.get("filename")), exist_ok=True) + logging.config.dictConfig(self.config) + + @classmethod + def create( + cls, + task_log_dir: str, + job_party_log_dir: Optional[str], + level: str, + delay: bool, + formatters: Optional[dict] = None, + ): + return FlowLogger( + config=LoggerConfigBuilder( + level, formatters, delay, task_log_dir, job_party_log_dir + ).build() + ) + + +class LoggerConfigBuilder: + def __init__(self, level, formatters, delay, log_base_dir, aggregate_log_base_dir): + self.version = 1 + self.formatters = formatters + if self.formatters is None: + default_format = ( + "[%(levelname)s][%(asctime)-8s][%(process)s][%(module)s.%(funcName)s][line:%(lineno)d]: %(message)s" + ) + self.formatters = { + "root": {"format": default_format}, + "component": {"format": default_format}, + } + self.handlers = {} + self.filters = {} + self.loggers = {} + self.root = { + "handlers": [], + "level": level, + } + self.disable_existing_loggers = False + + # add loggers + root_logger_dir = os.path.join(log_base_dir, "root") + self._add_root_loggers( + log_base_dir=root_logger_dir, formatter_name="root", delay=delay + ) + + component_logger_dir = os.path.join(log_base_dir, "component") + self._add_component_loggers( + log_base_dir=component_logger_dir, + formatter_name="component", + delay=delay, + loglevel=level, + ) + + if aggregate_log_base_dir is not None: + self._add_aggregate_error_logger( + aggregate_log_base_dir, formatter_name="root", delay=delay + ) + + def build(self): + return dict( + version=self.version, + formatters=self.formatters, + handlers=self.handlers, + filters=self.filters, + loggers=self.loggers, + root=self.root, + disable_existing_loggers=self.disable_existing_loggers, + ) + + def _add_root_loggers(self, log_base_dir, formatter_name, delay): + for level in _LOGGER_LEVELS: + handler_name = f"root_{level.lower()}" + self.handlers[handler_name] = self._create_file_handler( + level, formatter_name, delay, os.path.join(log_base_dir, level) + ) + self.root["handlers"].append(handler_name) + + def _add_aggregate_error_logger(self, log_base_dir, formatter_name, delay): + # error from all component + handler_name = "global_error" + self.handlers[handler_name] = self._create_file_handler( + "ERROR", formatter_name, delay, os.path.join(log_base_dir, "ERROR") + ) + self.root["handlers"].append(handler_name) + + def _add_component_loggers( + self, log_base_dir, formatter_name: str, loglevel: str, delay: bool + ): + # basic component logger handlers + # logger structure: + # component/ + # DEBUG + # INFO + # WARNING + # ERROR + component_handlers_names = [] + for level in _LOGGER_LEVELS: + handler_name = f"component_{level.lower()}" + self.handlers[handler_name] = self._create_file_handler( + level, formatter_name, delay, os.path.join(log_base_dir, level) + ) + component_handlers_names.append(handler_name) + + # add profile logger handler + # logger structure: + # component/ + # PROFILE + handler_name = "component_profile" + filter_name = "component_profile_filter" + self.filters[filter_name] = { + "name": "fate.arch.computing._profile", + "()": "logging.Filter", + } + self.handlers[handler_name] = self._create_file_handler( + "DEBUG", + formatter_name, + delay, + os.path.join(log_base_dir, "PROFILE"), + [filter_name], + ) + component_handlers_names.append(handler_name) + + # the "fate" name means the logger only log the logs from fate package + # so, don't change the name or the logger will not work + self.loggers["fate"] = dict( + handlers=component_handlers_names, + level=loglevel, + ) + + @staticmethod + def _create_file_handler(level, formatter, delay, filename, filters=None): + return { + "class": "logging.FileHandler", + "level": level, + "formatter": formatter, + "delay": delay, + "filename": filename, + "filters": [] if filters is None else filters, + } diff --git a/python/fate_flow/entity/spec/dag/_mlmd.py b/python/fate_flow/entity/spec/dag/_mlmd.py new file mode 100644 index 000000000..4fa98e83d --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_mlmd.py @@ -0,0 +1,29 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional, Any, Dict, Union + +import pydantic + + +class FlowMLMDMetadata(pydantic.BaseModel): + host: Optional[str] + port: Optional[int] + protocol: Optional[str] + + +class MLMDSpec(pydantic.BaseModel): + type: str + metadata: Union[Dict[str, Any], FlowMLMDMetadata] diff --git a/python/fate_flow/entity/spec/dag/_output.py b/python/fate_flow/entity/spec/dag/_output.py new file mode 100644 index 000000000..ce182e751 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_output.py @@ -0,0 +1,102 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Literal, Union, List, Dict, Optional + +import pydantic +from pydantic import typing + + +class MetricData(pydantic.BaseModel): + class Group(pydantic.BaseModel): + name: str + index: Optional[int] + name: str + type: Optional[str] + groups: List[Group] + step_axis: Optional[str] + data: Union[List, Dict] + + +class DirectoryDataPool(pydantic.BaseModel): + class DirectoryDataPoolMetadata(pydantic.BaseModel): + uri: str + format: str = "csv" + name_template: str = "{name}" # `name` and `uuid` allowed in template + + type: Literal["directory"] + metadata: DirectoryDataPoolMetadata + + +class CustomDataPool(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict + + +class DirectoryModelPool(pydantic.BaseModel): + class DirectoryDataPoolMetadata(pydantic.BaseModel): + uri: str + format: str = "json" + name_template: str = "{name}" # `name` and `uuid` allowed in template + + type: Literal["directory"] + metadata: DirectoryDataPoolMetadata + + +class CustomModelPool(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict + + +class DirectoryMetricPool(pydantic.BaseModel): + class DirectoryDataPoolMetadata(pydantic.BaseModel): + uri: str + format: str = "json" + name_template: str = "{name}" # `name` and `uuid` allowed in template + + type: Literal["directory"] + metadata: DirectoryDataPoolMetadata + + +class CustomMetricPool(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict + + +class OutputPoolConf(pydantic.BaseModel): + data: Union[DirectoryDataPool, CustomDataPool] + model: Union[DirectoryModelPool, CustomModelPool] + metric: Union[DirectoryMetricPool, CustomMetricPool] + + +class IOMeta(pydantic.BaseModel): + class InputMeta(pydantic.BaseModel): + data: typing.Dict[str, Union[List[Dict], Dict]] + model: typing.Dict[str, Union[List[Dict], Dict]] + + class OutputMeta(pydantic.BaseModel): + data: typing.Dict[str, Union[List[Dict], Dict]] + model: typing.Dict[str, Union[List[Dict], Dict]] + metric: typing.Dict[str, Union[List[Dict], Dict]] + + inputs: InputMeta + outputs: OutputMeta + + +class ComponentOutputMeta(pydantic.BaseModel): + class Status(pydantic.BaseModel): + code: int + exceptions: typing.Optional[str] + status: Status + io_meta: typing.Optional[IOMeta] diff --git a/python/fate_flow/entity/spec/dag/_party.py b/python/fate_flow/entity/spec/dag/_party.py new file mode 100644 index 000000000..cb8d9331e --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_party.py @@ -0,0 +1,25 @@ +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Union, Literal, List + +from pydantic import BaseModel + + +class PartySpec(BaseModel): + role: Union[Literal["guest", "host", "arbiter", "local"]] + party_id: List[str] + + def tuple(self): + return self.role, self.party_id diff --git a/python/fate_flow/entity/spec/dag/_task.py b/python/fate_flow/entity/spec/dag/_task.py new file mode 100644 index 000000000..7b6a78072 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_task.py @@ -0,0 +1,74 @@ +from typing import Optional, Union, Dict, Any, List + +import pydantic + +from fate_flow.entity.spec.dag._artifact import ArtifactInputApplySpec, ArtifactOutputApplySpec, \ + FlowRuntimeInputArtifacts +from fate_flow.entity.spec.dag._computing import StandaloneComputingSpec, SparkComputingSpec, EggrollComputingSpec +from fate_flow.entity.spec.dag._device import CPUSpec, GPUSpec +from fate_flow.entity.spec.dag._federation import StandaloneFederationSpec, RollSiteFederationSpec, RabbitMQFederationSpec,PulsarFederationSpec,OSXFederationSpec +from fate_flow.entity.spec.dag._logger import FlowLogger +from fate_flow.entity.spec.dag._mlmd import MLMDSpec + + +class TaskRuntimeConfSpec(pydantic.BaseModel): + device: Union[CPUSpec, GPUSpec] + computing: Union[StandaloneComputingSpec, EggrollComputingSpec, SparkComputingSpec] + storage: Optional[str] + federation: Union[ + StandaloneFederationSpec, + RollSiteFederationSpec, + RabbitMQFederationSpec, + PulsarFederationSpec, + OSXFederationSpec, + ] + logger: Union[FlowLogger] + + +class PreTaskConfigSpec(pydantic.BaseModel): + model_id: Optional[str] = "" + model_version: Optional[str] = "" + job_id: Optional[str] = "" + task_id: str + task_version: str + task_name: str + provider_name: str = "fate" + party_task_id: str + component: str + role: str + party_id: str + stage: str = "default" + parameters: Dict[str, Any] = {} + input_artifacts: FlowRuntimeInputArtifacts = {} + conf: TaskRuntimeConfSpec + mlmd: MLMDSpec + engine_run: Optional[Dict[str, Any]] = {} + computing_partitions: int = None + launcher_name: Optional[str] = "default" + launcher_conf: Optional[Dict] = {} + + +class TaskConfigSpec(pydantic.BaseModel): + job_id: Optional[str] = "" + task_id: str + party_task_id: str + task_name: str + component: str + role: str + party_id: str + stage: str = "default" + parameters: Dict[str, Any] = {} + input_artifacts: Optional[Dict[str, Union[List[ArtifactInputApplySpec], ArtifactInputApplySpec, None]]] = {} + output_artifacts: Optional[Dict[str, Union[ArtifactOutputApplySpec, None]]] = {} + conf: TaskRuntimeConfSpec + + +class TaskCleanupConfigSpec(pydantic.BaseModel): + computing: Union[StandaloneComputingSpec, EggrollComputingSpec, SparkComputingSpec] + federation: Union[ + StandaloneFederationSpec, + RollSiteFederationSpec, + RabbitMQFederationSpec, + PulsarFederationSpec, + OSXFederationSpec, + ] diff --git a/python/fate_flow/entity/spec/flow/__init__.py b/python/fate_flow/entity/spec/flow/__init__.py new file mode 100644 index 000000000..65c5cb9e4 --- /dev/null +++ b/python/fate_flow/entity/spec/flow/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ._model import MLModelSpec, Metadata +from ._storage import FileStorageSpec, MysqlStorageSpec, TencentCosStorageSpec +from ._provider import ProviderSpec, DockerProviderSpec, K8sProviderSpec, LocalProviderSpec +from ._scheduler import SchedulerInfoSpec + +__all__ = ["MLModelSpec", "FileStorageSpec", "MysqlStorageSpec", "TencentCosStorageSpec", "ProviderSpec", + "DockerProviderSpec", "K8sProviderSpec", "LocalProviderSpec", "SchedulerInfoSpec", "Metadata"] diff --git a/python/fate_flow/entity/model_spc.py b/python/fate_flow/entity/spec/flow/_model.py similarity index 84% rename from python/fate_flow/entity/model_spc.py rename to python/fate_flow/entity/spec/flow/_model.py index 29882d833..7ae73f07c 100644 --- a/python/fate_flow/entity/model_spc.py +++ b/python/fate_flow/entity/spec/flow/_model.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from datetime import datetime -from typing import List +from typing import List, Optional, Dict, Any, Union import pydantic @@ -57,3 +57,11 @@ class MLModelSpec(pydantic.BaseModel): federated: MLModelFederatedSpec party: MLModelPartySpec + + +class Metadata(pydantic.BaseModel): + metadata: dict + model_overview: MLModelSpec + model_key: str + index: Optional[Union[int, None]] = None + source: Optional[Dict[str, Any]] = None diff --git a/python/fate_flow/entity/spec/flow/_provider.py b/python/fate_flow/entity/spec/flow/_provider.py new file mode 100644 index 000000000..a61758c83 --- /dev/null +++ b/python/fate_flow/entity/spec/flow/_provider.py @@ -0,0 +1,59 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from typing import Optional, Union + +import pydantic + +from fate_flow.errors.server_error import FileNoFound + + +class BaseProvider(pydantic.BaseModel): + def __init__(self, check=False, **kwargs): + super(BaseProvider, self).__init__(**kwargs) + if check: + self.check() + + def check(self): + pass + + +class LocalProviderSpec(BaseProvider): + def check(self): + if not os.path.exists(self.path): + raise FileNoFound(path=self.path) + if self.venv and not os.path.exists(self.venv): + raise FileNoFound(venv=self.venv) + + path: str + venv: Optional[str] + + +class DockerProviderSpec(BaseProvider): + base_url: str + image: str + + +class K8sProviderSpec(BaseProvider): + image: str + namespace: str + config: Optional[dict] + + +class ProviderSpec(BaseProvider): + name: str + version: str + device: str + metadata: Union[LocalProviderSpec, DockerProviderSpec, K8sProviderSpec] diff --git a/python/fate_flow/entity/scheduler_structures.py b/python/fate_flow/entity/spec/flow/_scheduler.py similarity index 84% rename from python/fate_flow/entity/scheduler_structures.py rename to python/fate_flow/entity/spec/flow/_scheduler.py index 44332a46e..2acf82f17 100644 --- a/python/fate_flow/entity/scheduler_structures.py +++ b/python/fate_flow/entity/spec/flow/_scheduler.py @@ -12,14 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Union, Literal, Dict +from typing import Any, List, Union, Dict from pydantic import BaseModel - -class PartySpec(BaseModel): - role: Union[Literal["guest", "host", "arbiter"]] - party_id: List[Union[str, int]] +from fate_flow.entity.spec.dag import PartySpec class SchedulerInfoSpec(BaseModel): diff --git a/python/fate_flow/entity/spec/flow/_storage.py b/python/fate_flow/entity/spec/flow/_storage.py new file mode 100644 index 000000000..3177f458c --- /dev/null +++ b/python/fate_flow/entity/spec/flow/_storage.py @@ -0,0 +1,38 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Union + +import pydantic + + +class FileStorageSpec(pydantic.BaseModel): + path: Union[str, None] + + +class MysqlStorageSpec(pydantic.BaseModel): + name: str + user: str + passwd: str + host: str + port: int + max_connections: int + stale_timeout: int + + +class TencentCosStorageSpec(pydantic.BaseModel): + Region: str + SecretId: str + SecretKey: str + Bucket: str diff --git a/python/fate_flow/entity/types.py b/python/fate_flow/entity/types.py deleted file mode 100644 index 58c64cb52..000000000 --- a/python/fate_flow/entity/types.py +++ /dev/null @@ -1,110 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from enum import IntEnum, Enum - - -class CustomEnum(Enum): - @classmethod - def valid(cls, value): - try: - cls(value) - return True - except: - return False - - @classmethod - def values(cls): - return [member.value for member in cls.__members__.values()] - - @classmethod - def names(cls): - return [member.name for member in cls.__members__.values()] - - -class ProcessRole(CustomEnum): - DRIVER = "driver" - WORKER = "worker" - - -class ResourceOperation(CustomEnum): - APPLY = "apply" - RETURN = "return" - - -class CoordinationCommunicationProtocol(object): - HTTP = "http" - GRPC = "grpc" - - -class FederatedMode(object): - SINGLE = "SINGLE" - MULTIPLE = "MULTIPLE" - - def is_single(self, value): - return value == self.SINGLE - - def is_multiple(self, value): - return value == self.MULTIPLE - - -class KillProcessRetCode(IntEnum, CustomEnum): - KILLED = 0 - NOT_FOUND = 1 - ERROR_PID = 2 - - -class WorkerName(CustomEnum): - TASK_EXECUTOR = "task_executor" - TASK_INITIALIZER = "task_initializer" - PROVIDER_REGISTRAR = "provider_registrar" - DEPENDENCE_UPLOAD = "dependence_upload" - - -class ArtifactSourceType(object): - TASK_OUTPUT_ARTIFACT = "task_output_artifact" - MODEL_WAREHOUSE = "model_warehouse" - - -class Stage(object): - TRAIN = "train" - PREDICT = "predict" - DEFAULT = "default" - - -class ReturnCode: - - class Base: - SUCCESS = 0 - EXCEPTION_ERROR = 100 - - class Job: - NOT_FOUND = 1000 - CREATE_JOB_FAILED = 1001 - UPDATE_STATUS_FAILED = 1002 - UPDATE_FAILED = 1003 - KILL_FAILED = 1004 - APPLY_RESOURCE_FAILED = 1005 - - class Task: - NOT_FOUND = 2000 - START_FAILED = 2001 - UPDATE_STATUS_FAILED = 2002 - UPDATE_FAILED = 2003 - KILL_FAILED = 2004 - APPLY_RESOURCE_FAILED = 2005 - - class Site: - IS_STANDALONE = 3000 diff --git a/python/fate_flow/entity/types/__init__.py b/python/fate_flow/entity/types/__init__.py new file mode 100644 index 000000000..faa264a65 --- /dev/null +++ b/python/fate_flow/entity/types/__init__.py @@ -0,0 +1,28 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from ._work import * +from ._address import * +from ._engine import * +from ._output import * +from ._provider import * +from ._status import * +from ._federation import * +from ._command import * +from ._api import * +from ._instance import * +from ._permission import * +from ._input import * +from ._artificats import * diff --git a/python/fate_flow/engine/storage/_address.py b/python/fate_flow/entity/types/_address.py similarity index 83% rename from python/fate_flow/engine/storage/_address.py rename to python/fate_flow/entity/types/_address.py index cbc4d645f..62d1768fa 100644 --- a/python/fate_flow/engine/storage/_address.py +++ b/python/fate_flow/entity/types/_address.py @@ -31,6 +31,10 @@ def connector(self): def storage_engine(self): return + @property + def engine_path(self): + return + class StandaloneAddress(AddressBase): def __init__(self, home=None, name=None, namespace=None, storage_type=None, connector_name=None): @@ -53,6 +57,13 @@ def __repr__(self): def connector(self): return {"home": self.home} + @property + def engine_path(self): + if self.home: + return f"standalone:///{self.home}/{self.namespace}/{self.name}" + else: + return f"standalone:///{self.namespace}/{self.name}" + class EggRollAddress(AddressBase): def __init__(self, home=None, name=None, namespace=None, connector_name=None): @@ -74,6 +85,10 @@ def __repr__(self): def connector(self): return {"home": self.home} + @property + def engine_path(self): + return f"eggroll:///{self.namespace}/{self.name}" + class HDFSAddress(AddressBase): def __init__(self, name_node=None, path=None, connector_name=None): @@ -90,6 +105,16 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def engine_path(self): + if not self.name_node: + return f"hdfs://{self.path}" + else: + if "hdfs" not in self.name_node: + return f"hdfs://{self.name_node}{self.path}" + else: + return f"{self.name_node}{self.path}" + @property def connector(self): return {"name_node": self.name_node} @@ -109,6 +134,10 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def engine_path(self): + return f"file://{self.path}" + class ApiAddress(AddressBase): def __init__(self, method="POST", url=None, header=None, body=None, connector_name=None): @@ -127,6 +156,10 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def engine_path(self): + return self.url + class MysqlAddress(AddressBase): def __init__(self, user=None, passwd=None, host=None, port=None, db=None, name=None, connector_name=None): @@ -185,16 +218,20 @@ def connector(self): "database": self.database} -class LocalFSAddress(AddressBase): +class FileAddress(AddressBase): def __init__(self, path=None, connector_name=None): self.path = path - super(LocalFSAddress, self).__init__(connector_name=connector_name) + super(FileAddress, self).__init__(connector_name=connector_name) def __hash__(self): - return (self.path).__hash__() + return self.path.__hash__() def __str__(self): - return f"LocalFSAddress(path={self.path})" + return f"FileAddress(path={self.path})" def __repr__(self): return self.__str__() + + @property + def engine_path(self): + return f"file://{self.path}" diff --git a/python/fate_flow/entity/types/_api.py b/python/fate_flow/entity/types/_api.py new file mode 100644 index 000000000..14775030d --- /dev/null +++ b/python/fate_flow/entity/types/_api.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class AppType: + SITE = "site" + CLIENT = "client" diff --git a/python/fate_flow/entity/types/_artificats.py b/python/fate_flow/entity/types/_artificats.py new file mode 100644 index 000000000..fdd5520cf --- /dev/null +++ b/python/fate_flow/entity/types/_artificats.py @@ -0,0 +1,60 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import List + +from typing_extensions import Protocol + + +class ArtifactType(Protocol): + type_name: str + path_type: str + uri_types: List[str] + + +class DataframeArtifactType(ArtifactType): + type_name = "dataframe" + path_type = "distributed" + uri_types = ["eggroll", "hdfs"] + + +class TableArtifactType(ArtifactType): + type_name = "table" + path_type = "distributed" + uri_types = ["eggroll", "hdfs"] + + +class DataDirectoryArtifactType(ArtifactType): + type_name = "data_directory" + path_type = "directory" + uri_types = ["file"] + + +class ModelDirectoryArtifactType(ArtifactType): + type_name = "model_directory" + path_type = "directory" + uri_types = ["file"] + + +class JsonModelArtifactType(ArtifactType): + type_name = "json_model" + path_type = "file" + uri_types = ["file"] + + +class JsonMetricArtifactType(ArtifactType): + type_name = "json_metric" + path_type = "file" + uri_types = ["file"] diff --git a/python/fate_flow/entity/types/_command.py b/python/fate_flow/entity/types/_command.py new file mode 100644 index 000000000..6775a750f --- /dev/null +++ b/python/fate_flow/entity/types/_command.py @@ -0,0 +1,21 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity import CustomEnum + + +class ResourceOperation(CustomEnum): + APPLY = "apply" + RETURN = "return" diff --git a/python/fate_flow/entity/engine_types.py b/python/fate_flow/entity/types/_engine.py similarity index 90% rename from python/fate_flow/entity/engine_types.py rename to python/fate_flow/entity/types/_engine.py index 86e3bc483..ea68c6b81 100644 --- a/python/fate_flow/entity/engine_types.py +++ b/python/fate_flow/entity/types/_engine.py @@ -40,7 +40,7 @@ class StorageEngine(object): SIMPLE = "simple" PATH = "path" HIVE = "hive" - LOCALFS = "localfs" + FILE = "file" API = "api" @@ -53,5 +53,10 @@ class CoordinationProxyService(object): class FederatedCommunicationType(object): - PUSH = "PUSH" - PULL = "PULL" + POLL = "poll" + CALLBACK = "callback" + + +class LauncherType(object): + DEFAULT = "default" + DEEPSPEED = "deepspeed" diff --git a/python/fate_flow/entity/types/_federation.py b/python/fate_flow/entity/types/_federation.py new file mode 100644 index 000000000..7ca3d1c1c --- /dev/null +++ b/python/fate_flow/entity/types/_federation.py @@ -0,0 +1,30 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class CoordinationCommunicationProtocol(object): + HTTP = "http" + GRPC = "grpc" + + +class FederatedMode(object): + SINGLE = "SINGLE" + MULTIPLE = "MULTIPLE" + + def is_single(self, value): + return value == self.SINGLE + + def is_multiple(self, value): + return value == self.MULTIPLE + diff --git a/python/fate_flow/entity/types/_input.py b/python/fate_flow/entity/types/_input.py new file mode 100644 index 000000000..6cb21687e --- /dev/null +++ b/python/fate_flow/entity/types/_input.py @@ -0,0 +1,30 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class ArtifactSourceType(object): + TASK_OUTPUT_ARTIFACT = "task_output_artifact" + MODEL_WAREHOUSE = "model_warehouse" + DATA_WAREHOUSE = "data_warehouse" + + +class InputArtifactType(object): + DATA = "data" + MODEL = "model" + + @classmethod + def types(cls): + for _type in [cls.DATA, cls.MODEL]: + yield _type \ No newline at end of file diff --git a/python/fate_flow/entity/types/_instance.py b/python/fate_flow/entity/types/_instance.py new file mode 100644 index 000000000..13c708ef4 --- /dev/null +++ b/python/fate_flow/entity/types/_instance.py @@ -0,0 +1,45 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .._base import BaseEntity + + +class FlowInstance(BaseEntity): + def __init__(self, **kwargs): + self.instance_id = None, + self.timestamp = None, + self.version = None, + self.host = None, + self.grpc_port = None, + self.http_port = None + for k, v in kwargs.items(): + if hasattr(self, k): + setattr(self, k, v) + + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + if v is None: + continue + d[k] = v + return d + + @property + def grpc_address(self): + return f'{self.host}:{self.grpc_port}' + + @property + def http_address(self): + return f'{self.host}:{self.http_port}' diff --git a/python/fate_flow/entity/output_types.py b/python/fate_flow/entity/types/_output.py similarity index 84% rename from python/fate_flow/entity/output_types.py rename to python/fate_flow/entity/types/_output.py index 5cd2511ad..2fa8e9ee8 100644 --- a/python/fate_flow/entity/output_types.py +++ b/python/fate_flow/entity/types/_output.py @@ -24,3 +24,13 @@ class MetricData(BaseModel): groups: Dict metadata: Dict data: Union[List, Dict] + + +class ModelStorageEngine(object): + FILE = "file" + MYSQL = "mysql" + TENCENT_COS = "tencent_cos" + + +class ModelFileFormat(object): + JSON = "json" diff --git a/python/fate_flow/entity/types/_permission.py b/python/fate_flow/entity/types/_permission.py new file mode 100644 index 000000000..f88a8e346 --- /dev/null +++ b/python/fate_flow/entity/types/_permission.py @@ -0,0 +1,72 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json + +from fate_flow.entity import CustomEnum + + +class PermissionParameters(object): + def __init__(self, **kwargs): + self.party_id = None + self.component = None + self.dataset = {} + self.is_delete = False + for k, v in kwargs.items(): + if hasattr(self, k): + setattr(self, k, v) + + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + if v is None: + continue + d[k] = v + return d + + +class DataSet(object): + def __init__(self, namespace, name, **kwargs): + self.namespace = namespace + self.name = name + + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + if v is None: + continue + d[k] = str(v) + return d + + @property + def value(self): + return json.dumps(self.to_dict(), sort_keys=True) + + @property + def casbin_value(self): + return json.dumps(self.to_dict(), sort_keys=True, separators=(';', '-')) + + @staticmethod + def load_casbin_value(value): + return json.loads(value.replace(";", ",").replace("-", ":")) + + def check(self): + if not self.name or not self.namespace: + raise ValueError(f"name {self.name} or namespace {self.namespace} is null") + + +class PermissionType(CustomEnum): + COMPONENT = "component" + DATASET = "dataset" diff --git a/python/fate_flow/entity/types/_provider.py b/python/fate_flow/entity/types/_provider.py new file mode 100644 index 000000000..960158614 --- /dev/null +++ b/python/fate_flow/entity/types/_provider.py @@ -0,0 +1,23 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +class ProviderDevice(object): + LOCAL = "local" + DOCKER = "docker" + K8S = "k8s" + + +class ProviderName(object): + FATE = "fate" + FATE_FLOW = "fate_flow" diff --git a/python/fate_flow/entity/run_status.py b/python/fate_flow/entity/types/_status.py similarity index 93% rename from python/fate_flow/entity/run_status.py rename to python/fate_flow/entity/types/_status.py index 8d725e700..7880dac63 100644 --- a/python/fate_flow/entity/run_status.py +++ b/python/fate_flow/entity/types/_status.py @@ -67,7 +67,7 @@ class StateTransitionRule(BaseStateTransitionRule): StatusSet.RUNNING: [StatusSet.CANCELED, StatusSet.TIMEOUT, StatusSet.FAILED, StatusSet.SUCCESS], StatusSet.CANCELED: [StatusSet.WAITING], StatusSet.TIMEOUT: [StatusSet.FAILED, StatusSet.SUCCESS, StatusSet.WAITING], - StatusSet.FAILED: [StatusSet.WAITING], + StatusSet.FAILED: [StatusSet.WAITING, StatusSet.CANCELED], StatusSet.SUCCESS: [StatusSet.WAITING], } @@ -128,17 +128,3 @@ class SuccessStatus(BaseStatus): class AutoRerunStatus(BaseStatus): TIMEOUT = StatusSet.TIMEOUT FAILED = StatusSet.FAILED - - -class SchedulingStatusCode(object): - SUCCESS = 0 - NO_RESOURCE = 1 - PASS = 1 - NO_NEXT = 2 - HAVE_NEXT = 3 - FAILED = 4 - - -class FederatedSchedulingStatusCode(object): - SUCCESS = 0 - FAILED = 1 diff --git a/python/fate_flow/entity/types/_work.py b/python/fate_flow/entity/types/_work.py new file mode 100644 index 000000000..c5ff3bdec --- /dev/null +++ b/python/fate_flow/entity/types/_work.py @@ -0,0 +1,29 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity import CustomEnum + + +class ProcessRole(CustomEnum): + DRIVER = "driver" + WORKER = "worker" + + +class WorkerName(CustomEnum): + TASK_ENTRYPOINT = "task_entrypoint" + TASK_EXECUTE = "task_execute" + COMPONENT_DEFINE = "component_define" + TASK_CLEAN = "task_clean" + TASK_EXECUTE_CLEAN = "execute_clean" diff --git a/python/fate_flow/entrypoint/__init__.py b/python/fate_flow/entrypoint/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/entrypoint/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/entrypoint/cli.py b/python/fate_flow/entrypoint/cli.py new file mode 100644 index 000000000..f7fd047ec --- /dev/null +++ b/python/fate_flow/entrypoint/cli.py @@ -0,0 +1,112 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import os +import traceback + +import click + +from fate_flow.components.entrypoint.component import execute_component +from fate_flow.entity.spec.dag import PreTaskConfigSpec, TaskConfigSpec, TaskCleanupConfigSpec +from fate_flow.hub.flow_hub import FlowHub + + +@click.group() +def component(): + """ + Manipulate components: execute, list, generate describe file + """ + + +@component.command() +@click.option("--config", required=False, type=click.File(), help="config path") +@click.option("--env-name", required=False, type=str, help="env name for config") +@click.option("--wraps-module", required=False, type=str, help="component run wraps module") +def entrypoint(config, env_name, wraps_module): + # parse config + configs = {} + load_config_from_env(configs, env_name) + load_config_from_file(configs, config) + task_config = PreTaskConfigSpec.parse_obj(configs) + task_config.conf.logger.install() + logger = logging.getLogger(__name__) + logger.debug("logger installed") + logger.debug(f"task config: {task_config}") + FlowHub.load_components_wraps(config=task_config, module_name=wraps_module).run() + + +@component.command() +@click.option("--config", required=False, type=click.File(), help="config path") +@click.option("--env-name", required=False, type=str, help="env name for config") +@click.option("--wraps-module", required=False, type=str, help="component run wraps module") +def cleanup(config, env_name, wraps_module=None): + configs = {} + load_config_from_env(configs, env_name) + load_config_from_file(configs, config) + task_config = PreTaskConfigSpec.parse_obj(configs) + task_config.conf.logger.install() + logger = logging.getLogger(__name__) + logger.debug("logger installed") + logger.debug(f"task config: {task_config}") + FlowHub.load_components_wraps(config=task_config, module_name=wraps_module).cleanup() + + +@component.command() +@click.option("--config", required=False, type=click.File(), help="config path") +@click.option("--env-name", required=False, type=str, help="env name for config") +@click.option( + "--execution-final-meta-path", + type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True), + default=os.path.join(os.getcwd(), "execution_final_meta.json"), + show_default=True, + help="path for execution meta generated by component when execution finished", +) +def execute(config, env_name, execution_final_meta_path): + # parse config + configs = {} + load_config_from_env(configs, env_name) + load_config_from_file(configs, config) + task_config = TaskConfigSpec.parse_obj(configs) + task_config.conf.logger.install() + logger = logging.getLogger(__name__) + logger.debug("logger installed") + logger.debug(f"task config: {task_config}") + os.makedirs(os.path.dirname(execution_final_meta_path), exist_ok=True) + try: + execute_component(task_config) + with open(execution_final_meta_path, "w") as fw: + json.dump(dict(status=dict(code=0)), fw, indent=4) + except Exception as e: + with open(execution_final_meta_path, "w") as fw: + json.dump(dict(status=dict(code=-1, exceptions=traceback.format_exc())), fw) + raise e + + +def load_config_from_file(configs, config_file): + from ruamel import yaml + + if config_file is not None: + configs.update(yaml.safe_load(config_file)) + return configs + + +def load_config_from_env(configs, env_name): + import os + from ruamel import yaml + + if env_name is not None and os.environ.get(env_name): + configs.update(yaml.safe_load(os.environ[env_name])) + return configs diff --git a/python/fate_flow/worker/executor.py b/python/fate_flow/entrypoint/runner.py similarity index 80% rename from python/fate_flow/worker/executor.py rename to python/fate_flow/entrypoint/runner.py index e328347f2..f62e05204 100644 --- a/python/fate_flow/worker/executor.py +++ b/python/fate_flow/entrypoint/runner.py @@ -23,13 +23,11 @@ class Submit: @staticmethod def run(): import click - from fate.components.entrypoint.clean_cli import clean - from fate.components.entrypoint.component_cli import component + from fate_flow.entrypoint.cli import component cli = click.Group() cli.add_command(component) - cli.add_command(clean) - cli(prog_name="python -m fate.component") + cli(prog_name="python -m fate_flow.entrypoint") if __name__ == "__main__": diff --git a/python/fate_flow/errors/__init__.py b/python/fate_flow/errors/__init__.py new file mode 100644 index 000000000..e78b39576 --- /dev/null +++ b/python/fate_flow/errors/__init__.py @@ -0,0 +1,17 @@ +class FateFlowError(Exception): + code = None + message = 'Unknown Fate Flow Error' + + def __init__(self, message=None, **kwargs): + self.code = self.code + self.message = str(message) if message is not None else self.message + suffix = "" + if kwargs: + for k, v in kwargs.items(): + if v is not None and not callable(v): + if suffix: + suffix += "," + suffix += f"{k}[{v}]" + if suffix: + self.message += f": {suffix}" + super().__init__(self.code, self.message) diff --git a/python/fate_flow/errors/server_error.py b/python/fate_flow/errors/server_error.py new file mode 100644 index 000000000..ba012b82d --- /dev/null +++ b/python/fate_flow/errors/server_error.py @@ -0,0 +1,157 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.code import ReturnCode +from fate_flow.errors import FateFlowError + + +class JobParamsError(FateFlowError): + code = ReturnCode.Job.PARAMS_ERROR + message = 'Job params error' + + +class NoFoundJob(FateFlowError): + code = ReturnCode.Job.NOT_FOUND + message = 'No found job' + + +class FileNoFound(FateFlowError): + code = ReturnCode.File.FILE_NOT_FOUND + message = 'No found file or dir' + + +class CreateJobFailed(FateFlowError): + code = ReturnCode.Job.CREATE_JOB_FAILED + message = 'Create job failed' + + +class UpdateJobFailed(FateFlowError): + code = ReturnCode.Job.UPDATE_FAILED + message = 'Update job does not take effect' + + +class KillFailed(FateFlowError): + code = ReturnCode.Job.KILL_FAILED + message = "Kill job failed" + + +class JobResourceException(FateFlowError): + code = ReturnCode.Job.RESOURCE_EXCEPTION + message = "Job resource exception" + + +class InheritanceFailed(FateFlowError): + code = ReturnCode.Job.INHERITANCE_FAILED + message = "Inheritance job failed" + + +class NoFoundTask(FateFlowError): + code = ReturnCode.Task.NOT_FOUND + message = "No found task" + + +class StartTaskFailed(FateFlowError): + code = ReturnCode.Task.START_FAILED + message = "Start task failed" + + +class UpdateTaskFailed(FateFlowError): + code = ReturnCode.Task.UPDATE_FAILED + message = "Update task status does not take effect" + + +class KillTaskFailed(FateFlowError): + code = ReturnCode.Task.KILL_FAILED + message = 'Kill task failed' + + +class TaskResourceException(FateFlowError): + code = ReturnCode.Task.RESOURCE_EXCEPTION + message = "Task resource exception" + + +class NoFoundModelOutput(FateFlowError): + code = ReturnCode.Task.NO_FOUND_MODEL_OUTPUT + message = "No found output model" + + +class IsStandalone(FateFlowError): + code = ReturnCode.Site.IS_STANDALONE + message = "Site is standalone" + + +class DeviceNotSupported(FateFlowError): + code = ReturnCode.Provider.DEVICE_NOT_SUPPORTED + message = "Device not supported" + + +class RequestExpired(FateFlowError): + code = ReturnCode.API.EXPIRED + message = "Request has expired" + + +class InvalidParameter(FateFlowError): + code = ReturnCode.API.INVALID_PARAMETER + message = "Invalid parameter" + + +class NoFoundAppid(FateFlowError): + code = ReturnCode.API.NO_FOUND_APPID + message = "No found appid" + + +class ResponseException(FateFlowError): + code = ReturnCode.Server.RESPONSE_EXCEPTION + message = "Response exception" + + +class NoFoundServer(FateFlowError): + code = ReturnCode.Server.NO_FOUND + message = "No found server" + + +class NoFoundINSTANCE(FateFlowError): + code = ReturnCode.Server.NO_FOUND_INSTANCE + message = "No Found Flow Instance" + + +class NoFoundTable(FateFlowError): + code = ReturnCode.Table.NO_FOUND + message = "No found table" + + +class ExistsTable(FateFlowError): + code = ReturnCode.Table.EXISTS + message = "Exists table" + + +class NoPermission(FateFlowError): + code = ReturnCode.API.NO_PERMISSION + message = "No Permission" + + +class PermissionOperateError(FateFlowError): + code = ReturnCode.API.PERMISSION_OPERATE_ERROR + message = "Permission Operate Error" + + +class NoFoundFile(FateFlowError): + code = ReturnCode.API.NO_FOUND_FILE + message = "No Found File" + + +class RoleTypeError(FateFlowError): + code = ReturnCode.API.ROLE_TYPE_ERROR + message = "Role Type Error" diff --git a/python/fate_flow/errors/zookeeper_error.py b/python/fate_flow/errors/zookeeper_error.py new file mode 100644 index 000000000..67df7ac3d --- /dev/null +++ b/python/fate_flow/errors/zookeeper_error.py @@ -0,0 +1,24 @@ +from fate_flow.errors import FateFlowError + +__all__ = ['ServicesError', 'ServiceNotSupported', 'ZooKeeperNotConfigured', + 'MissingZooKeeperUsernameOrPassword', 'ZooKeeperBackendError'] + + +class ServicesError(FateFlowError): + message = 'Unknown services error' + + +class ServiceNotSupported(ServicesError): + message = 'The service {service_name} is not supported' + + +class ZooKeeperNotConfigured(ServicesError): + message = 'ZooKeeper has not been configured' + + +class MissingZooKeeperUsernameOrPassword(FateFlowError): + message = 'Using ACL for ZooKeeper is enabled but username or password is not configured' + + +class ZooKeeperBackendError(ServicesError): + message = 'ZooKeeper backend error: {error_message}' diff --git a/python/fate_flow/fate_flow_server.py b/python/fate_flow/fate_flow_server.py index e2cae7438..32d7a5cbc 100644 --- a/python/fate_flow/fate_flow_server.py +++ b/python/fate_flow/fate_flow_server.py @@ -13,78 +13,119 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# init env. must be the first import - import os import signal import sys import traceback import grpc -from grpc._cython import cygrpc from werkzeug.serving import run_simple from fate_flow.apps import app from fate_flow.controller.config_manager import ConfigManager +from fate_flow.hook import HookManager +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.manager.service.service_manager import service_db from fate_flow.runtime.runtime_config import RuntimeConfig from fate_flow.db.base_models import init_database_tables as init_flow_db from fate_flow.detection.detector import Detector, FederatedDetector from fate_flow.entity.types import ProcessRole from fate_flow.scheduler import init_scheduler from fate_flow.scheduler.job_scheduler import DAGScheduler -from fate_flow.settings import ( - GRPC_PORT, GRPC_SERVER_MAX_WORKERS, HOST, HTTP_PORT, detect_logger, stat_logger, +from fate_flow.runtime.system_settings import ( + GRPC_PORT, GRPC_SERVER_MAX_WORKERS, HOST, HTTP_PORT , GRPC_OPTIONS, FATE_FLOW_LOG_DIR, + LOG_LEVEL, ) from fate_flow.utils import process_utils from fate_flow.utils.grpc_utils import UnaryService, UnaryServiceOSX -from fate_flow.utils.log_utils import schedule_logger, getLogger +from fate_flow.utils.log import LoggerFactory, getLogger +from fate_flow.utils.log_utils import schedule_logger from fate_flow.utils.version import get_versions from fate_flow.utils.xthread import ThreadPoolExecutor from fate_flow.proto.rollsite import proxy_pb2_grpc from fate_flow.proto.osx import osx_pb2_grpc -if __name__ == '__main__': +detect_logger = getLogger("fate_flow_detect") +stat_logger = getLogger("fate_flow_stat") + + +def server_init(): + # init logs + LoggerFactory.set_directory(FATE_FLOW_LOG_DIR) + LoggerFactory.LEVEL = LOG_LEVEL + + # set signal + if "win" not in sys.platform.lower(): + signal.signal(signal.SIGCHLD, process_utils.wait_child_process) + # init db - signal.signal(signal.SIGCHLD, process_utils.wait_child_process) init_flow_db() - # init runtime config - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--version', default=False, help="fate flow version", action='store_true') - parser.add_argument('--debug', default=False, help="debug mode", action='store_true') - args = parser.parse_args() - if args.version: - print(get_versions()) - sys.exit(0) - # todo: add a general init steps? - RuntimeConfig.DEBUG = args.debug - if RuntimeConfig.DEBUG: - stat_logger.info("run on debug mode") + + # runtime config RuntimeConfig.init_env() RuntimeConfig.init_config(JOB_SERVER_HOST=HOST, HTTP_PORT=HTTP_PORT) RuntimeConfig.set_process_role(ProcessRole.DRIVER) + RuntimeConfig.init_config() + RuntimeConfig.set_service_db(service_db()) + RuntimeConfig.SERVICE_DB.register_flow() + + # manager ConfigManager.load() + HookManager.init() + AppManager.init() + + # scheduler init_scheduler() + + # detector Detector(interval=5 * 1000, logger=detect_logger).start() FederatedDetector(interval=10 * 1000, logger=detect_logger).start() DAGScheduler(interval=2 * 1000, logger=schedule_logger()).start() + + # provider register + ProviderManager.register_default_providers() + + +def start_server(debug=False): + # grpc thread_pool_executor = ThreadPoolExecutor(max_workers=GRPC_SERVER_MAX_WORKERS) - stat_logger.info(f"start grpc server thread pool by {thread_pool_executor._max_workers} max workers") + stat_logger.info(f"start grpc server thread pool by {thread_pool_executor.max_workers} max workers") server = grpc.server(thread_pool=thread_pool_executor, - options=[(cygrpc.ChannelArgKey.max_send_message_length, -1), - (cygrpc.ChannelArgKey.max_receive_message_length, -1)]) + options=GRPC_OPTIONS) + osx_pb2_grpc.add_PrivateTransferProtocolServicer_to_server(UnaryServiceOSX(), server) proxy_pb2_grpc.add_DataTransferServiceServicer_to_server(UnaryService(), server) server.add_insecure_port(f"{HOST}:{GRPC_PORT}") server.start() - print("FATE Flow grpc server start successfully") stat_logger.info("FATE Flow grpc server start successfully") - # start http server + # http + stat_logger.info("FATE Flow http server start...") + run_simple( + hostname=HOST, + port=HTTP_PORT, + application=app, + threaded=True, + use_reloader=debug, + use_debugger=debug + ) + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--version', default=False, help="fate flow version", action='store_true') + parser.add_argument('--debug', default=False, help="debug mode", action='store_true') + args = parser.parse_args() + if args.version: + print(get_versions()) + sys.exit(0) + + server_init() + try: - print("FATE Flow http server start...") - stat_logger.info("FATE Flow http server start...") - werkzeug_logger = getLogger("werkzeug") - run_simple(hostname=HOST, port=HTTP_PORT, application=app, threaded=True, use_reloader=RuntimeConfig.DEBUG, use_debugger=RuntimeConfig.DEBUG) - except Exception: + start_server(debug=args.debug) + except Exception as e: traceback.print_exc() + print(e) os.kill(os.getpid(), signal.SIGKILL) diff --git a/python/fate_flow/hook/__init__.py b/python/fate_flow/hook/__init__.py new file mode 100644 index 000000000..99abe9aef --- /dev/null +++ b/python/fate_flow/hook/__init__.py @@ -0,0 +1,67 @@ +import importlib + +from fate_flow.hook.common.parameters import SignatureParameters, AuthenticationParameters, PermissionCheckParameters, \ + SignatureReturn, AuthenticationReturn, PermissionReturn +from fate_flow.runtime.system_settings import HOOK_MODULE, CLIENT_AUTHENTICATION, SITE_AUTHENTICATION, PERMISSION_SWITCH +from fate_flow.entity.code import ReturnCode +from fate_flow.utils.log import getLogger + +stat_logger = getLogger() + + +class HookManager: + SITE_SIGNATURE = [] + SITE_AUTHENTICATION = [] + CLIENT_AUTHENTICATION = [] + PERMISSION_CHECK = [] + + @staticmethod + def init(): + if HOOK_MODULE is not None and (CLIENT_AUTHENTICATION or SITE_AUTHENTICATION or PERMISSION_SWITCH): + for modules in HOOK_MODULE.values(): + for module in modules.split(";"): + try: + importlib.import_module(module) + except Exception as e: + stat_logger.exception(e) + + @staticmethod + def register_site_signature_hook(func): + if SITE_AUTHENTICATION: + HookManager.SITE_SIGNATURE.append(func) + + @staticmethod + def register_site_authentication_hook(func): + HookManager.SITE_AUTHENTICATION.append(func) + + @staticmethod + def register_client_authentication_hook(func): + HookManager.CLIENT_AUTHENTICATION.append(func) + + @staticmethod + def register_permission_check_hook(func): + HookManager.PERMISSION_CHECK.append(func) + + @staticmethod + def client_authentication(parm: AuthenticationParameters) -> AuthenticationReturn: + if HookManager.CLIENT_AUTHENTICATION: + return HookManager.CLIENT_AUTHENTICATION[0](parm) + return AuthenticationReturn() + + @staticmethod + def site_signature(parm: SignatureParameters) -> SignatureReturn: + if HookManager.SITE_SIGNATURE: + return HookManager.SITE_SIGNATURE[0](parm) + return SignatureReturn() + + @staticmethod + def site_authentication(parm: AuthenticationParameters) -> AuthenticationReturn: + if HookManager.SITE_AUTHENTICATION: + return HookManager.SITE_AUTHENTICATION[0](parm) + return AuthenticationReturn() + + @staticmethod + def permission_check(parm: PermissionCheckParameters) -> PermissionReturn: + if HookManager.PERMISSION_CHECK: + return HookManager.PERMISSION_CHECK[0](parm) + return PermissionReturn() diff --git a/python/fate_flow/hook/common/__init__.py b/python/fate_flow/hook/common/__init__.py new file mode 100644 index 000000000..55fbe5eef --- /dev/null +++ b/python/fate_flow/hook/common/__init__.py @@ -0,0 +1,15 @@ + +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/hook/common/parameters.py b/python/fate_flow/hook/common/parameters.py new file mode 100644 index 000000000..db619f5bd --- /dev/null +++ b/python/fate_flow/hook/common/parameters.py @@ -0,0 +1,58 @@ +from fate_flow.entity.code import ReturnCode + + +class ParametersBase: + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + d[k] = v + return d + + +class AuthenticationParameters(ParametersBase): + def __init__(self, path, method, headers, form, data, json, full_path): + self.path = path + self.method = method + self.headers = headers + self.form = form + self.data = data + self.json = json + self.full_path = full_path + + +class AuthenticationReturn(ParametersBase): + def __init__(self, code=ReturnCode.Base.SUCCESS, message="success"): + self.code = code + self.message = message + + +class SignatureParameters(ParametersBase): + def __init__(self, party_id, body, initiator_party_id=""): + self.party_id = party_id + self.initiator_party_id = initiator_party_id + self.body = body + + +class SignatureReturn(ParametersBase): + def __init__(self, code=ReturnCode.Base.SUCCESS, signature=None, message=""): + self.code = code + self.signature = signature + self.message = message + + +class PermissionCheckParameters(ParametersBase): + def __init__(self, initiator_party_id, roles, component_list, dataset_list, dag_schema, component_parameters): + self.party_id = initiator_party_id + self.roles = roles + self.component_list = component_list + self.dataset_list = dataset_list + self.dag_schema = dag_schema + self.component_parameters = component_parameters + + +class PermissionReturn(ParametersBase): + def __init__(self, code=ReturnCode.Base.SUCCESS, message="success"): + self.code = code + self.message = message + + diff --git a/python/fate_flow/hook/flow/__init__.py b/python/fate_flow/hook/flow/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/hook/flow/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/hook/flow/client_authentication.py b/python/fate_flow/hook/flow/client_authentication.py new file mode 100644 index 000000000..18f97c16f --- /dev/null +++ b/python/fate_flow/hook/flow/client_authentication.py @@ -0,0 +1,34 @@ +from fate_flow.controller.app_controller import Authentication, PermissionController +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import InvalidParameter +from fate_flow.hook import HookManager +from fate_flow.hook.common.parameters import AuthenticationReturn, AuthenticationParameters + + +@HookManager.register_client_authentication_hook +def authentication(parm: AuthenticationParameters) -> AuthenticationReturn: + app_id = parm.headers.get("appId") + user_name = parm.headers.get("userName") + timestamp = parm.headers.get("Timestamp") + nonce = parm.headers.get("Nonce") + signature = parm.headers.get("Signature") + check_parameters(app_id, user_name, timestamp, nonce, signature) + if Authentication.md5_verify(app_id, timestamp, nonce, signature, user_name): + if PermissionController.enforcer(app_id, parm.path, parm.method): + return AuthenticationReturn(code=ReturnCode.Base.SUCCESS, message="success") + else: + return AuthenticationReturn(code=ReturnCode.API.AUTHENTICATION_FAILED, + message="Authentication Failed") + else: + return AuthenticationReturn(code=ReturnCode.API.VERIFY_FAILED, message="varify failed!") + + +def check_parameters(app_id, user_name, time_stamp, nonce, signature): + if not app_id: + raise InvalidParameter(name="appId") + if not time_stamp or not isinstance(time_stamp, str): + raise InvalidParameter(name="Timestamp") + if not nonce or not isinstance(time_stamp, str) or len(nonce) != 4: + raise InvalidParameter(name="Nonce") + if not signature: + raise InvalidParameter(name="Signature") diff --git a/python/fate_flow/hook/flow/permission.py b/python/fate_flow/hook/flow/permission.py new file mode 100644 index 000000000..1db64fcdf --- /dev/null +++ b/python/fate_flow/hook/flow/permission.py @@ -0,0 +1,22 @@ +from fate_flow.controller.permission_controller import PermissionCheck +from fate_flow.entity.code import ReturnCode +from fate_flow.hook import HookManager +from fate_flow.hook.common.parameters import PermissionCheckParameters, PermissionReturn +from fate_flow.runtime.system_settings import LOCAL_PARTY_ID, PARTY_ID + + +@HookManager.register_permission_check_hook +def permission(parm: PermissionCheckParameters) -> PermissionReturn: + if parm.party_id == LOCAL_PARTY_ID or parm.party_id == PARTY_ID: + return PermissionReturn() + + checker = PermissionCheck(**parm.to_dict()) + component_result = checker.check_component() + + if component_result.code != ReturnCode.Base.SUCCESS: + return component_result + + dataset_result = checker.check_dataset() + if dataset_result.code != ReturnCode.Base.SUCCESS: + return dataset_result + return PermissionReturn() diff --git a/python/fate_flow/hook/flow/site_authentication.py b/python/fate_flow/hook/flow/site_authentication.py new file mode 100644 index 000000000..70706a253 --- /dev/null +++ b/python/fate_flow/hook/flow/site_authentication.py @@ -0,0 +1,66 @@ +import hashlib + +from fate_flow.controller.app_controller import PermissionController, Authentication +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundAppid +from fate_flow.hook import HookManager +from fate_flow.hook.common.parameters import SignatureParameters, SignatureReturn, AuthenticationParameters, \ + AuthenticationReturn +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.runtime.system_settings import LOCAL_PARTY_ID, PARTY_ID + + +@HookManager.register_site_signature_hook +def signature(parm: SignatureParameters) -> SignatureReturn: + if parm.party_id == LOCAL_PARTY_ID: + parm.party_id = PARTY_ID + apps = AppManager.query_partner_app(party_id=parm.party_id) + if not apps: + e = NoFoundAppid(party_id=parm.party_id) + return SignatureReturn( + code=e.code, + message=e.message + ) + app = apps[0] + nonce = Authentication.generate_nonce() + timestamp = Authentication.generate_timestamp() + initiator_party_id = parm.initiator_party_id if parm.initiator_party_id else "" + key = hashlib.md5(str(app.f_app_id + initiator_party_id + nonce + timestamp).encode("utf8")).hexdigest().lower() + sign = hashlib.md5(str(key + app.f_app_token).encode("utf8")).hexdigest().lower() + + return SignatureReturn(signature={ + "Signature": sign, + "appId": app.f_app_id, + "Nonce": nonce, + "Timestamp": timestamp, + "initiatorPartyId": initiator_party_id + }) + + +@HookManager.register_site_authentication_hook +def authentication(parm: AuthenticationParameters) -> AuthenticationReturn: + app_id = parm.headers.get("appId") + timestamp = parm.headers.get("Timestamp") + nonce = parm.headers.get("Nonce") + sign = parm.headers.get("Signature") + initiator_party_id = parm.headers.get("initiatorPartyId") + check_parameters(app_id, timestamp, nonce, sign) + if Authentication.md5_verify(app_id, timestamp, nonce, sign, initiator_party_id): + if PermissionController.enforcer(app_id, parm.path, parm.method): + return AuthenticationReturn(code=ReturnCode.Base.SUCCESS, message="success") + else: + return AuthenticationReturn(code=ReturnCode.API.AUTHENTICATION_FAILED, + message=f"Authentication Failed: app_id[{app_id}, path[{parm.path}, method[{parm.method}]]]") + else: + return AuthenticationReturn(code=ReturnCode.API.VERIFY_FAILED, message="varify failed!") + + +def check_parameters(app_id, time_stamp, nonce, sign): + if not app_id: + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter: appId") + if not time_stamp or not isinstance(time_stamp, str): + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter:timeStamp") + if not nonce or not isinstance(time_stamp, str) or len(nonce) != 4: + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter: Nonce") + if not sign: + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter: Signature") diff --git a/python/fate_flow/hub/components_wraps/__init__.py b/python/fate_flow/hub/components_wraps/__init__.py new file mode 100644 index 000000000..c9fac8fd6 --- /dev/null +++ b/python/fate_flow/hub/components_wraps/__init__.py @@ -0,0 +1,26 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +from abc import ABCMeta + + +class WrapsABC(metaclass=ABCMeta): + @abc.abstractmethod + def run(self): + ... + + @abc.abstractmethod + def cleanup(self): + ... diff --git a/python/fate_flow/hub/components_wraps/fate/__init__.py b/python/fate_flow/hub/components_wraps/fate/__init__.py new file mode 100644 index 000000000..8d9cac80f --- /dev/null +++ b/python/fate_flow/hub/components_wraps/fate/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.hub.components_wraps.fate._wraps import FlowWraps + +__all__ = ["FlowWraps"] diff --git a/python/fate_flow/hub/components_wraps/fate/_wraps.py b/python/fate_flow/hub/components_wraps/fate/_wraps.py new file mode 100644 index 000000000..c17b7819a --- /dev/null +++ b/python/fate_flow/hub/components_wraps/fate/_wraps.py @@ -0,0 +1,584 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import json +import logging +import os.path +import tarfile +import traceback +from typing import List + +import yaml + +from fate_flow.engine.backend import build_backend +from fate_flow.engine.storage import StorageEngine +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.spec.dag import PreTaskConfigSpec, DataWarehouseChannelSpec, ComponentIOArtifactsTypeSpec, \ + TaskConfigSpec, ArtifactInputApplySpec, Metadata, RuntimeTaskOutputChannelSpec, \ + ArtifactOutputApplySpec, ModelWarehouseChannelSpec, ArtifactOutputSpec, ComponentOutputMeta, TaskCleanupConfigSpec + +from fate_flow.entity.types import DataframeArtifactType, TableArtifactType, TaskStatus, ComputingEngine, \ + JsonModelArtifactType + +from fate_flow.hub.components_wraps import WrapsABC +from fate_flow.manager.data.data_manager import DataManager, DatasetManager +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME, DEFAULT_OUTPUT_DATA_PARTITIONS +from fate_flow.utils import job_utils + +logger = logging.getLogger(__name__) + + +class FlowWraps(WrapsABC): + def __init__(self, config: PreTaskConfigSpec): + self.config = config + self.mlmd = self.load_mlmd(config.mlmd) + self.backend = build_backend(backend_name=self.config.conf.computing.type, launcher_name=self.config.launcher_name) + self._component_define = None + + @property + def task_info(self): + return { + "component": self.config.component, + "job_id": self.config.job_id, + "role": self.config.role, + "party_id": self.config.party_id, + "task_name": self.config.task_name, + "task_id": self.config.task_id, + "task_version": self.config.task_version + } + + @property + def task_input_dir(self): + return job_utils.get_task_directory(**self.task_info, input=True) + + @property + def task_output_dir(self): + return job_utils.get_task_directory(**self.task_info, output=True) + + def run(self): + code = 0 + exceptions = "" + try: + config = self.preprocess() + output_meta = self.run_component(config) + self.push_output(output_meta) + code = output_meta.status.code + exceptions = None + if output_meta.status.code != ReturnCode.Base.SUCCESS: + code = ReturnCode.Task.COMPONENT_RUN_FAILED + exceptions = output_meta.status.exceptions + logger.error(exceptions) + except Exception as e: + traceback.format_exc() + code = ReturnCode.Task.TASK_RUN_FAILED + exceptions = str(e) + logger.error(e) + finally: + self.report_status(code, exceptions) + + def cleanup(self): + config = TaskCleanupConfigSpec( + computing=self.config.conf.computing, + federation=self.config.conf.federation + ) + return self.backend.cleanup( + provider_name=self.config.provider_name, + config=config.dict(), + task_info=self.task_info, + party_task_id=self.config.party_task_id + ) + + def preprocess(self): + # input + logger.info("start generating input artifacts") + logger.info(self.config.input_artifacts) + input_artifacts = self._preprocess_input_artifacts() + logger.info("input artifacts are ready") + logger.debug(input_artifacts) + logger.info(f"PYTHON PATH: {os.environ.get('PYTHONPATH')}") + + # output + logger.info("start generating output artifacts") + output_artifacts = self._preprocess_output_artifacts() + logger.info(f"output_artifacts: {output_artifacts}") + config = TaskConfigSpec( + job_id=self.config.job_id, + task_id=self.config.task_id, + party_task_id=self.config.party_task_id, + component=self.config.component, + role=self.config.role, + party_id=self.config.party_id, + stage=self.config.stage, + parameters=self.config.parameters, + input_artifacts=input_artifacts, + output_artifacts=output_artifacts, + conf=self.config.conf, + task_name=self.config.task_name + ) + logger.debug(config) + return config + + def run_component(self, config): + self._set_env() + task_parameters = config.dict() + logger.info("start run task") + os.makedirs(self.task_input_dir, exist_ok=True) + os.makedirs(self.task_output_dir, exist_ok=True) + conf_path = os.path.join(self.task_input_dir, "task_parameters.yaml") + task_result = os.path.join(self.task_output_dir, "task_result.yaml") + with open(conf_path, "w") as f: + yaml.dump(task_parameters, f) + self.backend.run( + provider_name=self.config.provider_name, + task_info=self.task_info, + engine_run=self.config.engine_run, + launcher_conf=self.config.launcher_conf, + run_parameters=task_parameters, + output_path=task_result, + conf_path=conf_path, + session_id=self.config.party_task_id, + sync=True + ) + logger.info("finish task") + if os.path.exists(task_result): + with open(task_result, "r") as f: + try: + result = json.load(f) + output_meta = ComponentOutputMeta.parse_obj(result) + logger.debug(output_meta) + except: + logger.error(f"Task run failed, you can see the task result file for details: {task_result}") + else: + output_meta = ComponentOutputMeta(status=ComponentOutputMeta.Status( + code=ReturnCode.Task.NO_FOUND_RUN_RESULT, + exceptions=f"No found task output. Process exit code. " + )) + return output_meta + + def push_output(self, output_meta: ComponentOutputMeta): + if self.task_end_with_success(output_meta.status.code): + # push output data to server + if not output_meta.io_meta: + logger.info("No found io meta, pass push") + return + for key, datas in output_meta.io_meta.outputs.data.items(): + if isinstance(datas, list): + self._push_data(key, [ArtifactOutputSpec(**data) for data in datas]) + else: + self._push_data(key, [ArtifactOutputSpec(**datas)]) + + # push model + for key, models in output_meta.io_meta.outputs.model.items(): + if isinstance(models, list): + self._push_model(key, [ArtifactOutputSpec(**model) for model in models]) + else: + self._push_model(key, [ArtifactOutputSpec(**models)]) + + # push metric + for key, metrics in output_meta.io_meta.outputs.metric.items(): + if isinstance(metrics, list): + for metric in metrics: + output_metric = ArtifactOutputSpec(**metric) + self._push_metric(key, output_metric) + else: + output_metric = ArtifactOutputSpec(**metrics) + self._push_metric(key, output_metric) + + def _push_data(self, output_key, output_datas: List[ArtifactOutputSpec]): + logger.info("save data") + # logger.debug(f"key[{output_key}] output_datas[{output_datas}]") + for index, output_data in enumerate(output_datas): + if output_data.consumed is False: + # filter invalid output data + continue + namespace = output_data.metadata.namespace + name = output_data.metadata.name + if not namespace and not name: + namespace, name = DatasetManager.get_output_name(output_data.uri) + logger.info(f"save data tracking to {namespace}, {name}") + overview = output_data.metadata.data_overview + source = output_data.metadata.source + resp = self.mlmd.save_data_tracking( + execution_id=self.config.party_task_id, + output_key=output_key, + meta_data=output_data.metadata.metadata.get("schema", {}), + uri=output_data.uri, + namespace=namespace, + name=name, + overview=overview.dict() if overview else {}, + source=source.dict() if source else {}, + data_type=output_data.type_name, + index=index, + partitions=DEFAULT_OUTPUT_DATA_PARTITIONS + ) + self.log_response(resp, req_info="save data tracking") + + def _push_model(self, output_key, output_models: List[ArtifactOutputSpec]): + logger.info("save model") + logger.info(f"key[{output_key}] output_models[{output_models}]") + tar_io = io.BytesIO() + for output_model in output_models: + engine, address = DataManager.uri_to_address(output_model.uri) + if engine == StorageEngine.FILE: + _path = address.path + if os.path.exists(_path): + if os.path.isdir(_path): + path = _path + else: + path = os.path.dirname(_path) + model_key = os.path.basename(_path) + meta_path = os.path.join(path, f"{model_key}.meta.yaml") + with open(meta_path, "w") as fp: + output_model.metadata.model_key = model_key + output_model.metadata.index = output_model.metadata.source.output_index + output_model.metadata.type_name = output_model.type_name + yaml.dump(output_model.metadata.dict(), fp) + # tar and send to server + tar_io = self._tar_model(tar_io=tar_io, path=path) + type_name = output_model.type_name + else: + logger.warning(f"No found model path: {_path}") + else: + raise ValueError(f"Engine {engine} is not supported") + if output_models: + resp = self.mlmd.save_model( + model_id=self.config.model_id, + model_version=self.config.model_version, + execution_id=self.config.party_task_id, + output_key=output_key, + fp=tar_io, + type_name=type_name + ) + self.log_response(resp, req_info="save model") + + @staticmethod + def no_metadata_filter(tarinfo): + tarinfo.pax_headers = {} + return tarinfo + + @classmethod + def _tar_model(cls, tar_io, path): + with tarfile.open(fileobj=tar_io, mode="x:tar") as tar: + for _root, _dir, _files in os.walk(path): + for _f in _files: + full_path = os.path.join(_root, _f) + rel_path = os.path.relpath(full_path, path) + tar.add(full_path, rel_path, filter=cls.no_metadata_filter) + tar_io.seek(0) + return tar_io + + def _push_metric(self, output_key, output_metric: ArtifactOutputSpec): + logger.info(f"output metric: {output_metric}") + logger.info("save metric") + engine, address = DataManager.uri_to_address(output_metric.uri) + if engine == StorageEngine.FILE: + _path = address.path + if os.path.exists(_path): + with open(_path, "r") as f: + data = json.load(f) + if data: + resp = self.mlmd.save_metric( + execution_id=self.config.party_task_id, + data=data + ) + self.log_response(resp, req_info="save metric") + else: + logger.warning(f"No found metric path: {_path}") + else: + pass + + @staticmethod + def log_response(resp, req_info): + try: + logger.info(resp.json()) + resp_json = resp.json() + if resp_json.get("code") != ReturnCode.Base.SUCCESS: + logging.exception(f"{req_info}: {resp.text}") + except Exception: + logger.error(f"{req_info}: {resp.text}") + + def _preprocess_input_artifacts(self): + input_artifacts = {} + if self.config.input_artifacts.data: + for _k, _channels in self.config.input_artifacts.data.items(): + if isinstance(_channels, list): + input_artifacts[_k] = [] + for _channel in _channels: + _artifacts = self._intput_data_artifacts(_k, _channel) + if _artifacts: + input_artifacts[_k].append(_artifacts) + else: + input_artifacts[_k] = self._intput_data_artifacts(_k, _channels) + if not input_artifacts[_k]: + input_artifacts.pop(_k) + + if self.config.input_artifacts.model: + for _k, _channels in self.config.input_artifacts.model.items(): + if isinstance(_channels, list): + input_artifacts[_k] = [] + for _channel in _channels: + input_artifacts[_k].append(self._intput_model_artifacts(_k, _channel)) + else: + input_artifacts[_k] = self._intput_model_artifacts(_k, _channels) + if not input_artifacts[_k]: + input_artifacts.pop(_k) + return input_artifacts + + def _preprocess_output_artifacts(self): + # get component define + logger.debug("get component define") + define = self.component_define + logger.info(f"component define: {define}") + output_artifacts = {} + if not define: + return output_artifacts + else: + # data + for key in define.outputs.dict().keys(): + datas = getattr(define.outputs, key, None) + if datas: + for data in datas: + _output_artifacts = [] + for data_type in data.types: + _output_artifacts.append(self._output_artifacts(data_type.type_name, data.is_multi, + data.name, key)) + output_artifacts[data.name] = _output_artifacts[0] + return output_artifacts + + def _set_env(self): + if self.config.conf.computing.type == ComputingEngine.STANDALONE or \ + self.config.conf.federation.type == ComputingEngine.STANDALONE: + os.environ["STANDALONE_DATA_PATH"] = STANDALONE_DATA_HOME + + def _output_artifacts(self, type_name, is_multi, name, output_type=None): + output_artifacts = ArtifactOutputApplySpec(uri="", type_name=type_name) + if type_name in [DataframeArtifactType.type_name, TableArtifactType.type_name]: + uri = DatasetManager.output_data_uri(self.config.conf.storage, self.config.task_id, is_multi=is_multi) + else: + if output_type == "metric": + # api path + uri = self.mlmd.get_metric_save_url(execution_id=self.config.party_task_id) + else: + # local file path + uri = DatasetManager.output_local_uri(task_info=self.task_info, name=name, type_name=type_name, is_multi=is_multi) + output_artifacts.uri = uri + return output_artifacts + + @property + def component_define(self) -> ComponentIOArtifactsTypeSpec: + if not self._component_define: + self.set_component_define() + return self._component_define + + def set_component_define(self): + define = self.backend.get_component_define( + provider_name=self.config.provider_name, + task_info=self.task_info, + stage=self.config.stage + ) + if define: + self._component_define = ComponentIOArtifactsTypeSpec(**define) + + def _intput_data_artifacts(self, key, channel): + if self.config.role not in channel.roles: + logger.info(f"role {self.config.role} does not require intput data artifacts") + return + # data reference conversion + meta = ArtifactInputApplySpec( + metadata=Metadata( + metadata=dict(options=dict(partitions=self.config.computing_partitions)) + ), + uri="" + ) + query_field = {} + logger.info(f"get key[{key}] channel[{channel}]") + if isinstance(channel, DataWarehouseChannelSpec): + # external data reference -> data meta + if channel.name and channel.namespace: + query_field = { + "namespace": channel.namespace, + "name": channel.name + } + else: + query_field = { + "job_id": channel.job_id, + "role": self.config.role, + "party_id": self.config.party_id, + "task_name": channel.producer_task, + "output_key": channel.output_artifact_key + } + + elif isinstance(channel, RuntimeTaskOutputChannelSpec): + # this job output data reference -> data meta + query_field = { + "job_id": self.config.job_id, + "role": self.config.role, + "party_id": self.config.party_id, + "task_name": channel.producer_task, + "output_key": channel.output_artifact_key + } + logger.info(f"query data: [{query_field}]") + resp = self.mlmd.query_data_meta(**query_field) + logger.debug(resp.text) + resp_json = resp.json() + if resp_json.get("code") != 0: + # Judging whether to optional + for input_data_define in self.component_define.inputs.data: + if input_data_define.name == key and input_data_define.optional: + logger.info(f"component define input data name {key} optional {input_data_define.optional}") + return + raise ValueError(f"Get data artifacts failed: {query_field}, response: {resp.text}") + resp_data = resp_json.get("data", []) + logger.info(f"intput data artifacts are ready") + if len(resp_data) == 1: + data = resp_data[0] + schema = data.get("meta", {}) + meta.metadata.metadata.update({"schema": schema}) + + meta.uri = data.get("path") + source = data.get("source", {}) + if source: + meta.metadata.source = source + return meta + elif len(resp_data) > 1: + meta_list = [] + for data in resp_data: + schema = data.get("meta", {}) + meta.metadata.metadata.update({"schema": schema}) + meta.uri = data.get("path") + meta.type_name = data.get("data_type") + source = data.get("source", {}) + if source: + meta.metadata.source = source + meta_list.append(meta) + return meta_list + else: + raise RuntimeError(resp_data) + + def _intput_model_artifacts(self, key, channel): + if self.config.role not in channel.roles: + logger.info(f"role {self.config.role} does not require intput model artifacts") + return + # model reference conversion + meta = ArtifactInputApplySpec(metadata=Metadata(metadata={}), uri="") + query_field = { + "task_name": channel.producer_task, + "output_key": channel.output_artifact_key, + "role": self.config.role, + "party_id": self.config.party_id + } + logger.info(f"get key[{key}] channel[{channel}]") + if isinstance(channel, ModelWarehouseChannelSpec): + # external model reference -> download to local + if channel.model_id and channel.model_version: + query_field.update({ + "model_id": channel.model_id, + "model_version": channel.model_version + }) + else: + query_field.update({ + "model_id": self.config.model_id, + "model_version": self.config.model_version + }) + elif isinstance(channel, RuntimeTaskOutputChannelSpec): + query_field.update({ + "model_id": self.config.model_id, + "model_version": self.config.model_version + }) + + logger.info(f"query model: [{query_field}]") + + # this job output data reference -> data meta + input_model_base = os.path.join(self.task_input_dir, "model") + os.makedirs(input_model_base, exist_ok=True) + _io = io.BytesIO() + resp = self.mlmd.download_model(**query_field) + if resp.headers.get('content-type') == 'application/json': + raise RuntimeError(f"Download model failed, {resp.text}") + try: + for chunk in resp.iter_content(1024): + if chunk: + _io.write(chunk) + _io.seek(0) + model = tarfile.open(fileobj=_io) + except Exception as e: + for input_data_define in self.component_define.inputs.model: + if input_data_define.name == key and input_data_define.optional: + logger.info(f"component define input model name {key} optional {input_data_define.optional}") + return + raise RuntimeError(f"Download model failed: {query_field}") + logger.info(f"intput model artifacts are ready: {model.getnames()}") + metas = [] + file_names = model.getnames() + for name in file_names: + if name.endswith("yaml"): + fp = model.extractfile(name).read() + model_meta = yaml.safe_load(fp) + model_meta = Metadata.parse_obj(model_meta) + model_task_id = model_meta.source.task_id if model_meta.source.task_id else "" + input_model_file = os.path.join(input_model_base, "_".join([model_task_id, model_meta.model_key])) + if model_meta.type_name not in [JsonModelArtifactType.type_name]: + self._write_model_dir(model, input_model_file) + else: + model_fp = model.extractfile(model_meta.model_key).read() + with open(input_model_file, "wb") as fw: + fw.write(model_fp) + meta.uri = f"file://{input_model_file}" + meta.metadata = model_meta + metas.append(meta) + if not metas: + raise RuntimeError(f"Download model failed: {query_field}") + if len(metas) == 1: + return metas[0] + return metas + + @staticmethod + def _write_model_dir(model, path): + for name in model.getnames(): + if not name.endswith("yaml"): + model_fp = model.extractfile(name).read() + input_model_file = os.path.join(path, name) + os.makedirs(os.path.dirname(input_model_file), exist_ok=True) + with open(input_model_file, "wb") as fw: + fw.write(model_fp) + + def report_status(self, code, error=""): + if self.task_end_with_success(code): + resp = self.mlmd.report_task_status( + execution_id=self.config.party_task_id, + status=TaskStatus.SUCCESS + ) + else: + resp = self.mlmd.report_task_status( + execution_id=self.config.party_task_id, + status=TaskStatus.FAILED, + error=error + ) + self.log_response(resp, req_info="report status") + + @staticmethod + def task_end_with_success(code): + return code == 0 + + @staticmethod + def load_mlmd(mlmd): + if mlmd.type == "flow": + from ofx.api.client import FlowSchedulerApi + client = FlowSchedulerApi( + host=mlmd.metadata.get("host"), + port=mlmd.metadata.get("port"), + protocol=mlmd.metadata.get("protocol"), + api_version=mlmd.metadata.get("api_version")) + return client.worker diff --git a/python/fate_flow/hub/database/__init__.py b/python/fate_flow/hub/database/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/hub/database/mysql.py b/python/fate_flow/hub/database/mysql.py new file mode 100644 index 000000000..72bcc83cc --- /dev/null +++ b/python/fate_flow/hub/database/mysql.py @@ -0,0 +1,25 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from playhouse.pool import PooledMySQLDatabase + +from fate_flow.utils.password_utils import decrypt_database_config + + +def get_database_connection(config, decrypt_key): + database_config = config.copy() + db_name = database_config.pop("name") + decrypt_database_config(database_config, decrypt_key=decrypt_key) + return PooledMySQLDatabase(db_name, **database_config) diff --git a/python/fate_flow/hub/database/sqlite.py b/python/fate_flow/hub/database/sqlite.py new file mode 100644 index 000000000..024c67eca --- /dev/null +++ b/python/fate_flow/hub/database/sqlite.py @@ -0,0 +1,27 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from peewee import Insert + +from fate_flow.runtime.system_settings import SQLITE_PATH + + +def get_database_connection(config, decrypt_key): + Insert.on_conflict = lambda self, *args, **kwargs: self.on_conflict_replace() + from playhouse.apsw_ext import APSWDatabase + path = config.get("path") + if not path: + path = SQLITE_PATH + return APSWDatabase(path) diff --git a/python/fate_flow/hub/encrypt/__init__.py b/python/fate_flow/hub/encrypt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/hub/encrypt/password_encrypt.py b/python/fate_flow/hub/encrypt/password_encrypt.py new file mode 100644 index 000000000..664ee43b2 --- /dev/null +++ b/python/fate_flow/hub/encrypt/password_encrypt.py @@ -0,0 +1,41 @@ +import base64 + +from Crypto import Random +from Crypto.PublicKey import RSA +from Crypto.Cipher import PKCS1_v1_5 as PKCS1_cipher + + +def rsa_key_generate(): + random_generator = Random.new().read + rsa = RSA.generate(2048, random_generator) + private_pem = rsa.exportKey().decode() + public_pem = rsa.publickey().exportKey().decode() + with open('private_key.pem', "w") as f: + f.write(private_pem) + with open('public_key.pem', "w") as f: + f.write(public_pem) + return private_pem, public_pem + + +def encrypt_data(public_key, msg): + cipher = PKCS1_cipher.new(RSA.importKey(public_key)) + encrypt_text = base64.b64encode(cipher.encrypt(bytes(msg.encode("utf8")))) + return encrypt_text.decode('utf-8') + + +def pwdecrypt(private_key, encrypt_msg): + try: + cipher = PKCS1_cipher.new(RSA.importKey(private_key)) + back_text = cipher.decrypt(base64.b64decode(encrypt_msg), 0) + return back_text.decode('utf-8') + except Exception as e: + raise RuntimeError(f"passwd decrypt failed: {e}") + + +def test_encrypt_decrypt(): + msg = "fate" + private_key, public_key = rsa_key_generate() + encrypt_text = encrypt_data(public_key, msg) + print(encrypt_text) + decrypt_text = pwdecrypt(private_key, encrypt_text) + print(msg == decrypt_text) diff --git a/python/fate_flow/hub/flow_hub.py b/python/fate_flow/hub/flow_hub.py index 1e3656cdb..f66561d94 100644 --- a/python/fate_flow/hub/flow_hub.py +++ b/python/fate_flow/hub/flow_hub.py @@ -12,17 +12,53 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from fate_flow.entity.dag_structures import DAGSchema +# +from importlib import import_module + +from fate_flow.entity.types import ProviderName, ProviderDevice +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.runtime.system_settings import DEFAULT_JOB_PARSER_MODULE, DEFAULT_JOB_SCHEDULER_MODULE, \ + DEFAULT_COMPONENTS_WRAPS_MODULE class FlowHub: @staticmethod - def load_job_parser(dag): - if isinstance(dag, DAGSchema): - from fate_flow.hub.parser.default import JobParser - return JobParser(dag) + def load_job_parser(dag, module_name=DEFAULT_JOB_PARSER_MODULE): + class_name = module_name.split(".")[-1] + module = ".".join(module_name.split(".")[:-1]) + return getattr(import_module(module), class_name)(dag) + + @staticmethod + def load_job_scheduler(module_name=DEFAULT_JOB_SCHEDULER_MODULE): + class_name = module_name.split(".")[-1] + module = ".".join(module_name.split(".")[:-1]) + return getattr(import_module(module), class_name)() + + @staticmethod + def load_components_wraps(config, module_name=None): + if not module_name: + module_name = DEFAULT_COMPONENTS_WRAPS_MODULE + class_name = module_name.split(".")[-1] + module = ".".join(module_name.split(".")[:-1]) + return getattr(import_module(module), class_name)(config) + + @staticmethod + def load_provider_entrypoint(provider: ComponentProvider): + entrypoint = None + if provider.name == ProviderName.FATE and provider.device == ProviderDevice.LOCAL: + from fate_flow.hub.provider.fate import LocalFateEntrypoint + entrypoint = LocalFateEntrypoint(provider) + return entrypoint @staticmethod - def load_task_parser(*args, **kwargs): - from fate_flow.hub.parser.default import TaskParser - return TaskParser(*args, **kwargs) + def load_database(engine_name, config, decrypt_key): + try: + return getattr(import_module(f"fate_flow.hub.database.{engine_name}"), "get_database_connection")( + config, decrypt_key) + except Exception as e: + try: + import_module(f"fate_flow.hub.database.{engine_name}") + except: + raise SystemError(f"Not support database engine {engine_name}") + raise SystemError(f"load engine {engine_name} function " + f"fate_flow.hub.database.{engine_name}.get_database_connection failed: {e}") diff --git a/python/fate_flow/hub/parser/__init__.py b/python/fate_flow/hub/parser/__init__.py index da5c68180..7431348bb 100644 --- a/python/fate_flow/hub/parser/__init__.py +++ b/python/fate_flow/hub/parser/__init__.py @@ -32,10 +32,6 @@ def component_ref(self): def task_parameters(self): ... - @abc.abstractmethod - def update_runtime_artifacts(self, task_parameters): - ... - class JobParserABC(metaclass=ABCMeta): @property @@ -51,3 +47,19 @@ def infer_dependent_tasks(cls, task_input): @abc.abstractmethod def get_task_node(self, task_name): ... + + @property + def task_parser(self): + return TaskParserABC + + @abc.abstractmethod + def component_ref_list(self, role, party_id): + ... + + @abc.abstractmethod + def dataset_list(self, role, party_id): + ... + + @abc.abstractmethod + def role_parameters(self, role, party_id): + ... diff --git a/python/fate_flow/engine/abc/__init__.py b/python/fate_flow/hub/parser/fate/__init__.py similarity index 79% rename from python/fate_flow/engine/abc/__init__.py rename to python/fate_flow/hub/parser/fate/__init__.py index 8b2d3d6f9..ec9bae71f 100644 --- a/python/fate_flow/engine/abc/__init__.py +++ b/python/fate_flow/hub/parser/fate/__init__.py @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from fate_flow.engine.abc._storage import StorageSessionABC, StorageTableABC, StorageTableMetaABC +from fate_flow.hub.parser.fate._parser import TaskNodeInfo, JobParser, TaskParser __all__ = [ - "StorageSessionABC", "StorageTableABC", "StorageTableMetaABC" + "TaskNodeInfo", "JobParser", "TaskParser" ] + diff --git a/python/fate_flow/hub/parser/default/_parser.py b/python/fate_flow/hub/parser/fate/_parser.py similarity index 53% rename from python/fate_flow/hub/parser/default/_parser.py rename to python/fate_flow/hub/parser/fate/_parser.py index f32eda6ba..fd4b84bb4 100644 --- a/python/fate_flow/hub/parser/default/_parser.py +++ b/python/fate_flow/hub/parser/fate/_parser.py @@ -12,30 +12,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy import os +from typing import Dict, Union, List import networkx as nx -import copy - from pydantic import BaseModel -from typing import Dict, Union - -from ._federation import StandaloneFederationSpec, RollSiteFederationSpec, OSXFederationSpec, PulsarFederationSpec, \ - RabbitMQFederationSpec -from fate_flow.entity.dag_structures import ComponentSpec, RuntimeInputDefinition, ModelWarehouseChannelSpec, InputChannelSpec, DAGSchema,\ - RuntimeTaskOutputChannelSpec, TaskScheduleSpec, TaskRuntimeInputSpec, IOArtifact, OutputSpec, \ - OutputMetricSpec, OutputModelSpec, OutputDataSpec, MLMDSpec, LOGGERSpec, ComputingBackendSpec, \ - RuntimeConfSpec - -from fate_flow.entity.types import ArtifactSourceType -from fate_flow.manager.output_manager import OutputDataTracking -from fate_flow.operation.job_saver import JobSaver + +from fate_flow.entity.spec.dag import DataWarehouseChannelSpec, ModelWarehouseChannelSpec, \ + RuntimeTaskOutputChannelSpec, ComponentSpec, EggrollComputingSpec, SparkComputingSpec, StandaloneComputingSpec, \ + StandaloneFederationSpec, RollSiteFederationSpec, OSXFederationSpec, \ + PulsarFederationSpec, RabbitMQFederationSpec, FlowLogger, MLMDSpec, TaskRuntimeConfSpec, \ + DAGSchema, DAGSpec, PreTaskConfigSpec, FlowRuntimeInputArtifacts +from fate_flow.entity.types import EngineType, FederationEngine, DataSet, InputArtifactType, ArtifactSourceType, \ + ComputingEngine +from fate_flow.manager.service.provider_manager import ProviderManager from fate_flow.runtime.job_default_config import JobDefaultConfig -from fate_flow.settings import ENGINES, LOCAL_DATA_STORE_PATH, BASE_URI, PROXY, FATE_FLOW_CONF_PATH +from fate_flow.runtime.system_settings import ENGINES, PROXY, FATE_FLOW_CONF_PATH, HOST, HTTP_PORT, PROTOCOL, \ + API_VERSION from fate_flow.utils import job_utils, file_utils -from fate_flow.entity.engine_types import StorageEngine, EngineType, FederationEngine -from fate_flow.entity.scheduler_structures import SchedulerInfoSpec -from fate_flow.utils.log_utils import schedule_logger from .. import TaskParserABC, JobParserABC @@ -120,9 +115,11 @@ def conf(self, conf): class TaskParser(TaskParserABC): - def __init__(self, task_node, job_id, task_name, role, party_id, task_id="", execution_id="", - task_version=None, parties=None): + def __init__(self, task_node, job_id, task_name, role, party_id, task_id="", execution_id="", model_id="", + model_version="", task_version=None, parties=None, provider=None): self.task_node = task_node + self.model_id = model_id + self.model_version = model_version self.job_id = job_id self.task_name = task_name self.role = role @@ -131,6 +128,7 @@ def __init__(self, task_node, job_id, task_name, role, party_id, task_id="", exe self.task_version = task_version self.execution_id = execution_id self.parties = parties + self._provider = None @property def need_run(self): @@ -166,117 +164,81 @@ def output_definitions(self): @property def task_runtime_conf(self): - return self.task_node.conf + _rc = self.task_node.conf.get(self.role, {}).get(self.party_id, {}) + return _rc if _rc else {} @property - def input_parameters(self): - return self.task_node.runtime_parameters.get(self.role, {}).get(self.party_id, {}) + def task_runtime_launcher(self): + return self.task_runtime_conf.get("launcher", {}) @property - def input_artifacts(self): - task_artifacts = {} - if self.task_node.upstream_inputs: - for k, v in self.task_node.upstream_inputs.items(): - if isinstance(v, dict): - task_artifacts[k] = v - else: - _data = self.get_artifacts_data(k, v) - if _data: - task_artifacts[k] = _data - return task_artifacts - - def get_model_warehouse_source(self, channel: ModelWarehouseChannelSpec): - jobs = JobSaver.query_job(model_id=channel.model_id, model_version=channel.model_version, role=self.role, party_id=self.party_id) - if jobs: - job_id = jobs[0].f_job_id - return job_id - else: - raise Exception("no found model warehouse") - - def get_artifacts_data(self, name, channel: InputChannelSpec): - job_id = self.job_id - if isinstance(channel, ModelWarehouseChannelSpec): - job_id = self.get_model_warehouse_source(channel) - data = OutputDataTracking.query(task_name=channel.producer_task, output_key=channel.output_artifact_key, - role=self.role, party_id=self.party_id, job_id=job_id) - if data: - data = data[-1] - return IOArtifact(name=name, uri=data.f_uri, metadata=data.f_meta).dict() - return {} - - def generate_task_outputs(self): - return OutputSpec( - model=self.get_output_model_store_conf(), - data=self.get_output_data_store_conf(), - metric=self.get_output_data_metric_conf(), - ) - - def get_output_model_store_conf(self): - model_id, model_version = job_utils.generate_model_info(job_id=self.job_id) - _type = JobDefaultConfig.task_default_conf.get("output").get("model").get("type") - _format = JobDefaultConfig.task_default_conf.get("output").get("model").get("format") + def provider(self): + if not self._provider: + provider_name = self.task_runtime_conf.get("provider") + self._provider = ProviderManager.check_provider_name(provider_name) + return self._provider - return OutputModelSpec( - type=_type, - metadata={ - "uri": f"{BASE_URI}/worker/task/model/{self.job_id}/{self.role}/{self.party_id}/{model_id}/{str(model_version)}/{self.component_ref}/{self.task_name}", - "format": _format - } - ) - - def get_output_data_store_conf(self): - _type = JobDefaultConfig.task_default_conf.get("output").get("data").get("type") - _format = JobDefaultConfig.task_default_conf.get("output").get("data").get("format") - - if ENGINES.get(EngineType.STORAGE) in [StorageEngine.STANDALONE, StorageEngine.LOCALFS]: - os.makedirs(os.path.join(LOCAL_DATA_STORE_PATH, self.job_id, self.execution_id), exist_ok=True) - return OutputDataSpec(type=_type, metadata={ - "uri": f"file:///{LOCAL_DATA_STORE_PATH}/{self.job_id}/{self.execution_id}", - "format": _format - }) - elif ENGINES.get(EngineType.STORAGE) == StorageEngine.EGGROLL: - return OutputDataSpec(type=_type, metadata={ - "uri": f"eggroll:///output_data_{self.execution_id}", - "format": _format - }) - - def get_output_data_metric_conf(self): - _type = JobDefaultConfig.task_default_conf.get("output").get("metric").get("type") - _format = JobDefaultConfig.task_default_conf.get("output").get("metric").get("format") + @property + def provider_name(self): + return ProviderManager.parser_provider_name(self.provider)[0] - return OutputMetricSpec( - type=_type, - metadata={ - "uri": f"{BASE_URI}/worker/task/metric/{self.job_id}/{self.role}/" - f"{self.party_id}/{self.task_name}/{self.task_id}/{self.task_version}", - "format": _format - }) + @property + def input_parameters(self): + return self.task_node.runtime_parameters.get(self.role, {}).get(self.party_id, {}) @staticmethod def generate_mlmd(): _type = "flow" - _statu_uri = f"{BASE_URI}/worker/task/status" - _tracking_uri = f'{BASE_URI}/worker/task/output/tracking' return MLMDSpec( type=_type, metadata={ - "statu_uri": _statu_uri, - "tracking_uri": _tracking_uri + "host": HOST, + "port": HTTP_PORT, + "protocol": PROTOCOL, + "api_version": API_VERSION }) def generate_logger_conf(self): - logger_conf = JobDefaultConfig.task_default_conf.get("logger") - log_dir = job_utils.get_job_log_directory(self.job_id, self.role, self.party_id, self.task_name) - if logger_conf.get("metadata"): - logger_conf.get("metadata").update({"basepath": log_dir}) - return LOGGERSpec(**logger_conf) + logger_conf = JobDefaultConfig.task_logger + task_log_dir = job_utils.get_job_log_directory(self.job_id, self.role, self.party_id, self.task_name) + job_party_log_dir = job_utils.get_job_log_directory(self.job_id, self.role, self.party_id) + + # TODO: fix? + level = logger_conf.get("metadata", {}).get("level", "DEBUG") + delay = True + formatters = None + return FlowLogger.create(task_log_dir=task_log_dir, + job_party_log_dir=job_party_log_dir, + level=level, + delay=delay, + formatters=formatters) @staticmethod def generate_device(): - return JobDefaultConfig.task_default_conf.get("device") + return JobDefaultConfig.task_device def generate_computing_conf(self): - return ComputingBackendSpec(type=ENGINES.get(EngineType.COMPUTING).lower(), metadata={"computing_id": self.computing_id}) + if ENGINES.get(EngineType.COMPUTING).lower() == ComputingEngine.STANDALONE: + return StandaloneComputingSpec( + type=ENGINES.get(EngineType.COMPUTING).lower(), + metadata={"computing_id": self.computing_id} + ) + + if ENGINES.get(EngineType.COMPUTING).lower() == ComputingEngine.EGGROLL: + return EggrollComputingSpec( + type=ENGINES.get(EngineType.COMPUTING).lower(), + metadata={"computing_id": self.computing_id} + ) + + if ENGINES.get(EngineType.COMPUTING).lower() == ComputingEngine.SPARK: + return SparkComputingSpec( + type=ENGINES.get(EngineType.COMPUTING).lower(), + metadata={"computing_id": self.computing_id} + ) + + @staticmethod + def generate_storage_conf(): + return ENGINES.get(EngineType.STORAGE).lower() def generate_federation_conf(self): parties_info = [] @@ -312,8 +274,10 @@ def generate_federation_conf(self): federation_id=self.federation_id, parties=parties, route_table=PulsarFederationSpec.MetadataSpec.RouteTable( - route={k: PulsarFederationSpec.MetadataSpec.RouteTable.Route(**v) for k, v in route_table.items() if k!= "default"}, - default=PulsarFederationSpec.MetadataSpec.RouteTable.Default(**route_table.get("default", {})) if route_table.get("default") else None + route={k: PulsarFederationSpec.MetadataSpec.RouteTable.Route(**v) for k, v in route_table.items() if + k != "default"}, + default=PulsarFederationSpec.MetadataSpec.RouteTable.Default( + **route_table.get("default", {})) if route_table.get("default") else None ), pulsar_config=PulsarFederationSpec.MetadataSpec.PulsarConfig(**proxy_conf) )) @@ -332,33 +296,35 @@ def generate_federation_conf(self): @property def task_conf(self): - return RuntimeConfSpec( - output=self.generate_task_outputs(), - mlmd=self.generate_mlmd(), + return TaskRuntimeConfSpec( logger=self.generate_logger_conf(), device=self.generate_device(), computing=self.generate_computing_conf(), - federation=self.generate_federation_conf() + federation=self.generate_federation_conf(), + storage=self.generate_storage_conf() ) @property - def task_parameters(self) -> TaskScheduleSpec: - return TaskScheduleSpec( + def task_parameters(self) -> PreTaskConfigSpec: + return PreTaskConfigSpec( + model_id=self.model_id, + model_version=self.model_version, + job_id=self.job_id, task_id=self.task_id, + task_version=self.task_version, + task_name=self.task_name, + provider_name=self.provider_name, party_task_id=self.execution_id, component=self.component_ref, role=self.role, stage=self.stage, party_id=self.party_id, - inputs=TaskRuntimeInputSpec(parameters=self.input_parameters).dict(), - conf=self.task_conf + parameters=self.input_parameters, + input_artifacts=self.task_node.upstream_inputs.get(self.role).get(self.party_id), + conf=self.task_conf, + mlmd=self.generate_mlmd() ) - def update_runtime_artifacts(self, task_parameters): - task_parameters["inputs"].update({"artifacts": self.input_artifacts}) - schedule_logger(job_id=self.job_id).info(f"update artifacts: {self.input_artifacts}") - return task_parameters - class JobParser(JobParserABC): def __init__(self, dag_conf): @@ -384,7 +350,8 @@ def parse_dag(self, dag_schema: DAGSchema, component_specs: Dict[str, ComponentS if not task_spec.conf: task_conf = copy.deepcopy(job_conf) else: - task_conf = copy.deepcopy(task_spec.conf).update(job_conf) + task_conf = copy.deepcopy(job_conf) + task_conf.update(task_spec.conf) if task_spec.stage: task_stage = task_spec.stage @@ -395,61 +362,114 @@ def parse_dag(self, dag_schema: DAGSchema, component_specs: Dict[str, ComponentS self._tasks[name].component_spec = component_specs[name] self._init_task_runtime_parameters_and_conf(name, dag_schema, task_conf) - if not task_spec.inputs or not task_spec.inputs.artifacts: + self._init_upstream_inputs(name, dag_schema.dag) + + def _init_upstream_inputs(self, name, dag: DAGSpec): + task_spec = dag.tasks[name] + common_upstream_inputs = dict() + if task_spec.inputs: + common_upstream_inputs = self._get_upstream_inputs(name, task_spec) + + upstream_inputs = dict() + role_keys = set([party.role for party in dag.parties]) + for party in dag.parties: + if party.role not in role_keys: + continue + upstream_inputs[party.role] = dict() + for party_id in party.party_id: + upstream_inputs[party.role][party_id] = copy.deepcopy(common_upstream_inputs) + + party_tasks = dag.party_tasks + if not party_tasks: + self._tasks[name].upstream_inputs = upstream_inputs + return + + for site_name, party_tasks_spec in party_tasks.items(): + if not party_tasks_spec.tasks or name not in party_tasks_spec.tasks: + continue + party_task_spec = party_tasks_spec.tasks[name] + if not party_task_spec.inputs: + continue + party_upstream_inputs = self._get_upstream_inputs(name, party_task_spec) + for party in party_tasks_spec.parties: + for party_id in party.party_id: + upstream_inputs[party.role][party_id].update(party_upstream_inputs) + + self._tasks[name].upstream_inputs = upstream_inputs + + def _get_upstream_inputs(self, name, task_spec): + upstream_inputs = dict() + runtime_roles = self._tasks[name].runtime_roles + input_artifacts = task_spec.inputs + + for input_type in InputArtifactType.types(): + artifacts = getattr(input_artifacts, input_type) + if not artifacts: continue - upstream_inputs = dict() - runtime_roles = self._tasks[name].runtime_roles - for input_key, output_specs_dict in task_spec.inputs.artifacts.items(): - upstream_inputs[input_key] = dict() + upstream_inputs[input_type] = dict() + + for input_key, output_specs_dict in artifacts.items(): + upstream_inputs[input_type][input_key] = dict() for artifact_source, channel_spec_list in output_specs_dict.items(): if artifact_source == ArtifactSourceType.MODEL_WAREHOUSE: if isinstance(channel_spec_list, list): inputs = [] for channel in channel_spec_list: - model_warehouse_channel = ModelWarehouseChannelSpec(**channel.dict(exclude_defaults=True)) + model_warehouse_channel = ModelWarehouseChannelSpec( + **channel.dict(exclude_defaults=True)) if model_warehouse_channel.model_id is None: - model_warehouse_channel.model_id = self._conf.get("model_id", None) - model_warehouse_channel.model_version = self._conf.get("model_version", None) + model_warehouse_channel.model_id = \ + self._conf.get("model_warehouse", {}).get("model_id", None) + model_warehouse_channel.model_version = \ + self._conf.get("model_warehouse", {}).get("model_version", None) inputs.append(model_warehouse_channel) else: inputs = ModelWarehouseChannelSpec(**channel_spec_list.dict(exclude_defaults=True)) if inputs.model_id is None: - inputs.model_id = self._conf.get("model_id", None) - inputs.model_version = self._conf.get("model_version", None) + inputs.model_id = self._conf.get("model_warehouse", {}).get("model_id", None) + inputs.model_version = self._conf.get("model_warehouse", {}).get("model_version", None) - upstream_inputs[input_key] = inputs + upstream_inputs[input_type][input_key] = inputs continue else: + if artifact_source == ArtifactSourceType.DATA_WAREHOUSE: + channel_spec = DataWarehouseChannelSpec + else: + channel_spec = RuntimeTaskOutputChannelSpec if isinstance(channel_spec_list, list): - inputs = [RuntimeTaskOutputChannelSpec(**channel.dict(exclude_defaults=True)) + inputs = [channel_spec(**channel.dict(exclude_defaults=True)) for channel in channel_spec_list] else: - inputs = RuntimeTaskOutputChannelSpec(**channel_spec_list.dict(exclude_defaults=True)) + inputs = channel_spec(**channel_spec_list.dict(exclude_defaults=True)) - upstream_inputs[input_key] = inputs + upstream_inputs[input_type][input_key] = inputs if not isinstance(channel_spec_list, list): channel_spec_list = [channel_spec_list] for channel_spec in channel_spec_list: - dependent_task = channel_spec.producer_task - self._add_edge(dependent_task, name) + if isinstance(channel_spec, RuntimeTaskOutputChannelSpec): + dependent_task = channel_spec.producer_task + self._add_edge(dependent_task, name) - upstream_inputs = self.check_and_add_runtime_roles(upstream_inputs, runtime_roles) - self._tasks[name].upstream_inputs = upstream_inputs + upstream_inputs = self.check_and_add_runtime_roles(upstream_inputs, runtime_roles) + return upstream_inputs @staticmethod def check_and_add_runtime_roles(upstream_inputs, runtime_roles): correct_inputs = copy.deepcopy(upstream_inputs) - for input_key, channel_list in upstream_inputs.items(): - if isinstance(channel_list, list): - for idx, channel in enumerate(channel_list): - if channel.roles is None: - correct_inputs[input_key][idx].roles = runtime_roles - else: - if channel_list.roles is None: - correct_inputs[input_key].roles = runtime_roles + for input_type in InputArtifactType.types(): + if input_type not in upstream_inputs: + continue + for input_key, channel_list in upstream_inputs[input_type].items(): + if isinstance(channel_list, list): + for idx, channel in enumerate(channel_list): + if channel.roles is None: + correct_inputs[input_type][input_key][idx].roles = runtime_roles + else: + if channel_list.roles is None: + correct_inputs[input_type][input_key].roles = runtime_roles return correct_inputs @@ -468,8 +488,8 @@ def _init_task_runtime_parameters_and_conf(self, task_name: str, dag_schema: DAG role_keys = role_keys & task_role_keys common_parameters = dict() - if task_spec.inputs and task_spec.inputs.parameters: - common_parameters = task_spec.inputs.parameters + if task_spec.parameters: + common_parameters = task_spec.parameters task_parameters = dict() task_conf = dict() @@ -488,26 +508,24 @@ def _init_task_runtime_parameters_and_conf(self, task_name: str, dag_schema: DAG if dag.party_tasks: party_tasks = dag.party_tasks for site_name, party_tasks_spec in party_tasks.items(): - if task_name not in party_tasks_spec.tasks: - continue + if party_tasks_spec.conf: + for party in party_tasks_spec.parties: + if party.role in task_parameters: + for party_id in party.party_id: + task_conf[party.role][party_id].update(party_tasks_spec.conf) - party_task_conf = copy.deepcopy(party_tasks_spec.conf) if party_tasks_spec.conf else dict() - party_task_conf.update(global_task_conf) + if not party_tasks_spec.tasks or task_name not in party_tasks_spec.tasks: + continue party_parties = party_tasks_spec.parties party_task_spec = party_tasks_spec.tasks[task_name] - if party_task_spec.conf: - _conf = copy.deepcopy(party_task_spec.conf) - party_task_conf = _conf.update(party_task_conf) - for party in party_parties: - if party.role in task_parameters: - for party_id in party.party_id: - task_conf[party.role][party_id].update(party_task_conf) - - if not party_task_spec.inputs: - continue - parameters = party_task_spec.inputs.parameters + for party in party_parties: + if party.role in task_parameters: + for party_id in party.party_id: + task_conf[party.role][party_id].update(party_task_spec.conf) + + parameters = party_task_spec.parameters if parameters: for party in party_parties: @@ -526,39 +544,73 @@ def topological_sort(self): return nx.topological_sort(self._dag) @classmethod - def infer_dependent_tasks(cls, task_input: RuntimeInputDefinition): - if not task_input or not task_input.artifacts: + def infer_dependent_tasks(cls, input_artifacts): + print(input_artifacts) + if not input_artifacts: return [] + dependent_task_list = list() - for artifact_name, artifact_channel in task_input.artifacts.items(): - for artifact_source_type, channels in artifact_channel.items(): - if artifact_source_type == ArtifactSourceType.MODEL_WAREHOUSE: - continue + for input_type in InputArtifactType.types(): + artifacts = getattr(input_artifacts, input_type) + if not artifacts: + continue + for artifact_name, artifact_channel in artifacts.items(): + for artifact_source_type, channels in artifact_channel.items(): + if artifact_source_type in [ArtifactSourceType.MODEL_WAREHOUSE, ArtifactSourceType.DATA_WAREHOUSE]: + continue + + if not isinstance(channels, list): + channels = [channels] + for channel in channels: + dependent_task_list.append(channel.producer_task) - if not isinstance(channels, list): - channels = [channels] - for channel in channels: - dependent_task_list.append(channel.producer_task) return dependent_task_list + @property + def task_parser(self): + return TaskParser + + def component_ref_list(self, role, party_id): + _list = [] + for name in self.topological_sort(): + node = self.get_task_node(name) + if node: + if self.task_parser( + task_node=self.get_task_node(task_name=name), + job_id="", task_name=name, role=role, party_id=party_id + ).need_run: + _list.append(node.component_ref) + return _list + + def dataset_list(self, role, party_id) -> List[DataSet]: + def append_dataset(datasets, channel): + if isinstance(channel, DataWarehouseChannelSpec): + if channel.name and channel.namespace: + datasets.append(DataSet(**{ + "name": channel.name, + "namespace": channel.namespace + })) + _list = [] + for task_name in self.topological_sort(): + task_node = self.get_task_node(task_name) + input_artifacts = FlowRuntimeInputArtifacts(**task_node.upstream_inputs.get(role, {}).get(party_id, {})) + if input_artifacts.data: + for _k, _channels in input_artifacts.data.items(): + if isinstance(_channels, list): + for _channel in _channels: + append_dataset(_list, _channel) + else: + append_dataset(_list, _channels) + return _list + + def role_parameters(self, role, party_id): + _dict = {} + for task_name in self.topological_sort(): + task_node = self.get_task_node(task_name) + _dict[task_node.component_ref] = task_node.runtime_parameters.get(role, {}).get(party_id, {}) + return _dict + class Party(BaseModel): role: str party_id: Union[str, int] - - -class DagSchemaParser(object): - def __init__(self, dag_schema): - self.dag_schema = DAGSchema(**dag_schema) - - @property - def job_schedule_info(self) -> SchedulerInfoSpec: - return SchedulerInfoSpec( - dag=self.dag_schema.dict(), - parties=[party.dict() for party in self.dag_schema.dag.parties], - initiator_party_id=self.dag_schema.dag.conf.initiator_party_id, - scheduler_party_id=self.dag_schema.dag.conf.scheduler_party_id, - federated_status_collect_type=self.dag_schema.dag.conf.federated_status_collect_type, - model_id=self.dag_schema.dag.conf.model_id, - model_version=self.dag_schema.dag.conf.model_version - ) diff --git a/python/fate_flow/hub/provider/__init__.py b/python/fate_flow/hub/provider/__init__.py new file mode 100644 index 000000000..1b32724e6 --- /dev/null +++ b/python/fate_flow/hub/provider/__init__.py @@ -0,0 +1,22 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +from typing import Dict + + +class EntrypointABC: + @abc.abstractmethod + def component_list(self) -> Dict: + ... diff --git a/python/fate_flow/hub/provider/fate.py b/python/fate_flow/hub/provider/fate.py new file mode 100644 index 000000000..6977d8525 --- /dev/null +++ b/python/fate_flow/hub/provider/fate.py @@ -0,0 +1,33 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + +from fate_flow.hub.provider import EntrypointABC + + +class LocalFateEntrypoint(EntrypointABC): + def __init__(self, provider): + self.provider = provider + + @property + def component_list(self): + if self.provider.python_path and self.provider.python_path not in sys.path: + sys.path.append(self.provider.python_path) + from fate.components.core import list_components + # {'buildin': [], 'thirdparty': []} + components = list_components() + _list = components.get('buildin', []) + _list.extend(components.get("thirdparty", [])) + return _list diff --git a/python/fate_flow/hub/scheduler/__init__.py b/python/fate_flow/hub/scheduler/__init__.py new file mode 100644 index 000000000..13e79cd27 --- /dev/null +++ b/python/fate_flow/hub/scheduler/__init__.py @@ -0,0 +1,65 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +from typing import Dict + + +class JobSchedulerABC: + @classmethod + def submit(cls, dag_schema) -> Dict: + """ + description： + Create a job to all parties and set the job status to waiting + :param dag_schema: job config; + + """ + + @abc.abstractmethod + def run_do(self): + """ + description： + Scheduling various status job, including: waiting、running、ready、rerun、end、etc. + """ + + @classmethod + def stop_job(cls, job_id: str, stop_status: str): + """ + description： + Stop a job to all parties and set the job status to end status + :param job_id: job id + :param stop_status: In which state to stop the task. + + """ + + @classmethod + def rerun_job(cls, job_id: str, auto: bool, tasks=None): + """ + description： + rerun a job + :param job_id: job id + :param auto: Whether the scheduler automatically rerun + :param tasks: Specified rerun task list. + + """ + + @classmethod + def adapt_party_parameters(cls, dag_schema, role): + """ + """ + + @classmethod + def check_job_parameters(cls, dag_schema, is_local): + """ + """ diff --git a/python/fate_flow/hub/parser/default/__init__.py b/python/fate_flow/hub/scheduler/fate/__init__.py similarity index 79% rename from python/fate_flow/hub/parser/default/__init__.py rename to python/fate_flow/hub/scheduler/fate/__init__.py index 891be154f..8ac1db19b 100644 --- a/python/fate_flow/hub/parser/default/__init__.py +++ b/python/fate_flow/hub/scheduler/fate/__init__.py @@ -12,10 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from fate_flow.hub.parser.default._parser import TaskNodeInfo, JobParser, TaskParser, DagSchemaParser +from fate_flow.hub.scheduler.fate._scheduler import DAGScheduler __all__ = [ - "TaskNodeInfo", "JobParser", "TaskParser", "DagSchemaParser" + "DAGScheduler" ] diff --git a/python/fate_flow/hub/scheduler/fate/_scheduler.py b/python/fate_flow/hub/scheduler/fate/_scheduler.py new file mode 100644 index 000000000..c6b57d142 --- /dev/null +++ b/python/fate_flow/hub/scheduler/fate/_scheduler.py @@ -0,0 +1,640 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from copy import deepcopy + +from pydantic import typing + +from fate_flow.controller.job_controller import JobInheritance +from fate_flow.controller.task_controller import TaskController +from fate_flow.entity.code import SchedulingStatusCode, FederatedSchedulingStatusCode +from fate_flow.entity.spec.dag import DAGSchema, JobConfSpec +from fate_flow.db.schedule_models import ScheduleJob, ScheduleTaskStatus +from fate_flow.entity.types import StatusSet, JobStatus, TaskStatus, EndStatus, InterruptStatus, ResourceOperation, \ + FederatedCommunicationType, AutoRerunStatus, ComputingEngine, EngineType +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundJob, JobParamsError +from fate_flow.hub.flow_hub import FlowHub +from fate_flow.hub.scheduler import JobSchedulerABC +from fate_flow.manager.model.model_meta import ModelMeta +from fate_flow.operation.job_saver import ScheduleJobSaver +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.runtime.system_settings import ENGINES, COMPUTING_CONF, IGNORE_RESOURCE_ROLES, PARTY_ID, LOCAL_PARTY_ID +from fate_flow.scheduler.federated_scheduler import FederatedScheduler +from fate_flow.utils import job_utils, schedule_utils, wraps_utils +from fate_flow.utils.base_utils import json_dumps +from fate_flow.utils.log_utils import schedule_logger, exception_to_trace_string + + +class DAGScheduler(JobSchedulerABC): + @classmethod + def check_job_parameters(cls, dag_schema: DAGSchema, is_local: bool = False): + if not dag_schema.dag.conf: + dag_schema.dag.conf = JobConfSpec() + dag_schema.dag.conf.initiator_party_id = PARTY_ID + if not dag_schema.dag.conf.scheduler_party_id: + if not is_local: + dag_schema.dag.conf.scheduler_party_id = PARTY_ID + else: + dag_schema.dag.conf.scheduler_party_id = LOCAL_PARTY_ID + if not dag_schema.dag.conf.computing_partitions: + dag_schema.dag.conf.computing_partitions = JobDefaultConfig.computing_partitions + + # check inheritance + JobInheritance.check(dag_schema.dag.conf.inheritance) + + # check model warehouse + model_warehouse = dag_schema.dag.conf.model_warehouse + if model_warehouse: + if not ModelMeta.query(model_id=model_warehouse.model_id, model_version=model_warehouse.model_version): + raise JobParamsError( + model_id=model_warehouse.model_id, + model_version=model_warehouse.model_version, + position="dag_schema.dag.conf.model_warehouse" + ) + + @classmethod + def submit(cls, dag_schema): + dag_schema = DAGSchema(**dag_schema) + job_id = job_utils.generate_job_id() + schedule_logger(job_id).info( + f"submit job, dag {dag_schema.dag.dict()}, schema version {dag_schema.schema_version}") + submit_result = { + "job_id": job_id, + "data": {} + } + try: + job = ScheduleJob() + job.f_job_id = job_id + job.f_parties = [party.dict() for party in dag_schema.dag.parties] + job.f_initiator_party_id = dag_schema.dag.conf.initiator_party_id + job.f_scheduler_party_id = dag_schema.dag.conf.scheduler_party_id + if dag_schema.dag.conf.priority: + job.f_priority = dag_schema.dag.conf.priority + cls.fill_default_job_parameters(job_id, dag_schema) + job.f_dag = dag_schema.dict() + submit_result["data"].update({ + "model_id": dag_schema.dag.conf.model_id, + "model_version": dag_schema.dag.conf.model_version + }) + job.f_status = StatusSet.READY + ScheduleJobSaver.create_job(job.to_human_model_dict()) + status_code, response = FederatedScheduler.create_job( + job_id, job.f_parties, job.f_initiator_party_id, {"dag_schema": dag_schema.dict(), "job_id": job_id} + ) + if status_code != FederatedSchedulingStatusCode.SUCCESS: + job.f_status = JobStatus.FAILED + FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, job_info={ + "job_id": job.f_job_id, + "status": job.f_status + }) + raise Exception("create job failed", response) + else: + job.f_status = JobStatus.WAITING + TaskController.create_schedule_tasks(job, dag_schema) + status_code, response = FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, + "status": job.f_status}) + if status_code != FederatedSchedulingStatusCode.SUCCESS: + raise Exception(f"set job to waiting status failed: {response}") + ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": job.f_status}) + schedule_logger(job_id).info(f"submit job successfully, job id is {job.f_job_id}") + result = { + "code": ReturnCode.Base.SUCCESS, + "message": "success" + } + submit_result.update(result) + except Exception as e: + schedule_logger(job_id).exception(e) + submit_result["code"] = ReturnCode.Job.CREATE_JOB_FAILED + submit_result["message"] = exception_to_trace_string(e) + return submit_result + + @classmethod + def fill_default_job_parameters(cls, job_id: str, dag_schema: DAGSchema): + if not dag_schema.dag.conf.sync_type: + dag_schema.dag.conf.sync_type = JobDefaultConfig.sync_type + if not dag_schema.dag.conf.model_id or not dag_schema.dag.conf.model_id: + dag_schema.dag.conf.model_id, dag_schema.dag.conf.model_version = job_utils.generate_model_info(job_id) + if not dag_schema.dag.conf.auto_retries: + dag_schema.dag.conf.auto_retries = JobDefaultConfig.auto_retries + + @classmethod + def adapt_party_parameters(cls, dag_schema: DAGSchema, role): + cores, task_run, task_cores = cls.calculate_resource(dag_schema, role) + job_info = {"cores": cores, "remaining_cores": cores} + if dag_schema.dag.conf.inheritance: + job_info.update({"inheritance": dag_schema.dag.conf.inheritance.dict()}) + return job_info, task_run, task_cores + + @classmethod + def calculate_resource(cls, dag_schema: DAGSchema, role): + cores = dag_schema.dag.conf.cores if dag_schema.dag.conf.cores else JobDefaultConfig.job_cores + if dag_schema.dag.conf.task and dag_schema.dag.conf.task.run: + task_run = dag_schema.dag.conf.task.run + else: + task_run = {} + task_cores = cores + default_task_run = deepcopy(JobDefaultConfig.task_run.get(ENGINES.get(EngineType.COMPUTING), {})) + if ENGINES.get(EngineType.COMPUTING) == ComputingEngine.SPARK: + if "num-executors" not in task_run: + task_run["num-executors"] = default_task_run.get("num-executors") + if "executor-cores" not in task_run: + task_run["executor-cores"] = default_task_run.get("executor-cores") + if role in IGNORE_RESOURCE_ROLES: + task_run["num-executors"] = 1 + task_run["executor-cores"] = 1 + task_cores = int(task_run.get("num-executors")) * (task_run.get("executor-cores")) + if task_cores > cores: + cores = task_cores + if ENGINES.get(EngineType.COMPUTING) == ComputingEngine.EGGROLL: + if "eggroll.session.processors.per.node" not in task_run: + task_run["eggroll.session.processors.per.node"] = \ + default_task_run.get("eggroll.session.processors.per.node") + task_cores = int(task_run.get("eggroll.session.processors.per.node")) * COMPUTING_CONF.get( + ComputingEngine.EGGROLL).get("nodes") + if task_cores > cores: + cores = task_cores + if role in IGNORE_RESOURCE_ROLES: + task_run["eggroll.session.processors.per.node"] = 1 + if ENGINES.get(EngineType.COMPUTING) == ComputingEngine.STANDALONE: + if "cores" not in task_run: + task_run["cores"] = default_task_run.get("cores") + task_cores = int(task_run.get("cores")) + if task_cores > cores: + cores = task_cores + if role in IGNORE_RESOURCE_ROLES: + task_run["cores"] = 1 + if role in IGNORE_RESOURCE_ROLES: + cores = 0 + task_cores = 0 + return cores, task_run, task_cores + + def run_do(self): + # waiting + schedule_logger().info("start schedule waiting jobs") + # order by create_time and priority + jobs = ScheduleJobSaver.query_job( + status=JobStatus.WAITING, + order_by=["priority", "create_time"], + reverse=[True, False] + ) + schedule_logger().info(f"have {len(jobs)} waiting jobs") + if len(jobs): + job = jobs[0] + schedule_logger().info(f"schedule waiting job {job.f_job_id}") + try: + self.schedule_waiting_jobs(job=job, lock=True) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error("schedule waiting job failed") + schedule_logger().info("schedule waiting jobs finished") + + # running + schedule_logger().info("start schedule running jobs") + jobs = ScheduleJobSaver.query_job(status=JobStatus.RUNNING, order_by="create_time", reverse=False) + schedule_logger().info(f"have {len(jobs)} running jobs") + for job in jobs: + schedule_logger().info(f"schedule running job {job.f_job_id}") + try: + self.schedule_running_job(job=job, lock=True) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error("schedule job failed") + schedule_logger().info("schedule running jobs finished") + + # ready + schedule_logger().info("start schedule ready jobs") + jobs = ScheduleJobSaver.query_job(ready_signal=True, order_by="create_time", reverse=False) + schedule_logger().info(f"have {len(jobs)} ready jobs") + for job in jobs: + schedule_logger().info(f"schedule ready job {job.f_job_id}") + try: + pass + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error(f"schedule ready job failed:\n{e}") + schedule_logger().info("schedule ready jobs finished") + + # rerun + schedule_logger().info("start schedule rerun jobs") + jobs = ScheduleJobSaver.query_job(rerun_signal=True, order_by="create_time", reverse=False) + schedule_logger().info(f"have {len(jobs)} rerun jobs") + for job in jobs: + schedule_logger(job.f_job_id).info(f"schedule rerun job {job.f_job_id}") + try: + self.schedule_rerun_job(job=job) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error("schedule job failed") + schedule_logger().info("schedule rerun jobs finished") + + @classmethod + def apply_job_resource(cls, job): + apply_status_code, federated_response = FederatedScheduler.resource_for_job( + job_id=job.f_job_id, + roles=job.f_parties, + operation_type=ResourceOperation.APPLY.value + ) + if apply_status_code == FederatedSchedulingStatusCode.SUCCESS: + return True + else: + cls.rollback_job_resource(job, federated_response) + return False + + @classmethod + def rollback_job_resource(cls, job, federated_response): + rollback_party = [] + failed_party = [] + for dest_role in federated_response.keys(): + for dest_party_id in federated_response[dest_role].keys(): + retcode = federated_response[dest_role][dest_party_id]["code"] + if retcode == ReturnCode.Base.SUCCESS: + rollback_party.append({"role": dest_role, "party_id": [dest_party_id]}) + else: + failed_party.append({"role": dest_role, "party_id": [dest_party_id]}) + schedule_logger(job.f_job_id).info("job apply resource failed on {}, rollback {}".format(failed_party, + rollback_party)) + if rollback_party: + return_status_code, federated_response = FederatedScheduler.resource_for_job( + job_id=job.f_job_id, + roles=rollback_party, + operation_type=ResourceOperation.RETURN.value + ) + if return_status_code != FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job.f_job_id).info(f"job return resource failed:\n{federated_response}") + else: + schedule_logger(job.f_job_id).info("job no party should be rollback resource") + + @classmethod + @wraps_utils.schedule_lock + def schedule_waiting_jobs(cls, job: ScheduleJob): + if job.f_cancel_signal: + FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, "status": JobStatus.CANCELED}) + ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": JobStatus.CANCELED}) + schedule_logger(job.f_job_id).info("job have cancel signal") + return + status = cls.apply_job_resource(job) + if status: + cls.start_job(job_id=job.f_job_id, roles=job.f_parties) + + @wraps_utils.schedule_lock + def schedule_running_job(self, job: ScheduleJob, force_sync_status=False): + schedule_logger(job.f_job_id).info("scheduling running job") + task_scheduling_status_code, auto_rerun_tasks, tasks = TaskScheduler.schedule(job=job) + tasks_status = dict([(task.f_task_name, task.f_status) for task in tasks]) + schedule_logger(job_id=job.f_job_id).info(f"task_scheduling_status_code: {task_scheduling_status_code}, " + f"tasks_status: {tasks_status.values()}") + new_job_status = self.calculate_job_status(task_scheduling_status_code=task_scheduling_status_code, + tasks_status=tasks_status.values()) + if new_job_status == JobStatus.WAITING and job.f_cancel_signal: + new_job_status = JobStatus.CANCELED + total, finished_count = self.calculate_job_progress(tasks_status=tasks_status) + new_progress = float(finished_count) / total * 100 + schedule_logger(job.f_job_id).info( + f"job status is {new_job_status}, calculate by task status list: {tasks_status}") + if new_job_status != job.f_status or new_progress != job.f_progress: + # Make sure to update separately, because these two fields update with anti-weight logic + if int(new_progress) - job.f_progress > 0: + job.f_progress = new_progress + FederatedScheduler.update_job(job_id=job.f_job_id, + roles=job.f_parties, + command_body={"job_id": job.f_job_id, "progress": job.f_progress}) + self.update_job_on_scheduler(schedule_job=job, update_fields=["progress"]) + if new_job_status != job.f_status: + job.f_status = new_job_status + FederatedScheduler.sync_job_status( + job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, "status": new_job_status} + ) + self.update_job_on_scheduler(schedule_job=job, update_fields=["status"]) + if EndStatus.contains(job.f_status): + self.finish(job=job, end_status=job.f_status) + if auto_rerun_tasks: + schedule_logger(job.f_job_id).info("job have auto rerun tasks") + self.rerun_job(job_id=job.f_job_id, tasks=auto_rerun_tasks, auto=True) + if force_sync_status: + FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_roles, status=job.f_status, + job_info=job.to_human_model_dict()) + schedule_logger(job.f_job_id).info("finish scheduling running job") + + @wraps_utils.schedule_lock + def schedule_rerun_job(self, job): + if EndStatus.contains(job.f_status): + job.f_status = JobStatus.WAITING + schedule_logger(job.f_job_id).info("job has been finished, set waiting to rerun") + status, response = FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, + "status": job.f_status}) + if status == FederatedSchedulingStatusCode.SUCCESS: + schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) + schedule_logger(job.f_job_id).info("job set waiting to rerun successfully") + else: + schedule_logger(job.f_job_id).warning("job set waiting to rerun failed") + ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": job.f_status}) + else: + schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) + self.schedule_running_job(job) + + @classmethod + def calculate_job_status(cls, task_scheduling_status_code, tasks_status): + tmp_status_set = set(tasks_status) + if TaskStatus.PASS in tmp_status_set: + tmp_status_set.remove(TaskStatus.PASS) + tmp_status_set.add(TaskStatus.SUCCESS) + if len(tmp_status_set) == 1: + return tmp_status_set.pop() + else: + if TaskStatus.RUNNING in tmp_status_set: + return JobStatus.RUNNING + if TaskStatus.WAITING in tmp_status_set: + if task_scheduling_status_code == SchedulingStatusCode.HAVE_NEXT: + return JobStatus.RUNNING + else: + pass + for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), + reverse=True): + if status in tmp_status_set: + return status + if tmp_status_set == {TaskStatus.WAITING, + TaskStatus.SUCCESS} and task_scheduling_status_code == SchedulingStatusCode.NO_NEXT: + return JobStatus.CANCELED + + raise Exception("calculate job status failed, all task status: {}".format(tasks_status)) + + @classmethod + def calculate_job_progress(cls, tasks_status): + total = 0 + finished_count = 0 + for task_status in tasks_status.values(): + total += 1 + if EndStatus.contains(task_status): + finished_count += 1 + return total, finished_count + + @classmethod + def start_job(cls, job_id, roles): + schedule_logger(job_id).info(f"start job {job_id}") + status_code, response = FederatedScheduler.start_job(job_id, roles) + schedule_logger(job_id).info(f"start job {job_id} status code: {status_code}, response: {response}") + ScheduleJobSaver.update_job_status(job_info={"job_id": job_id, "status": StatusSet.RUNNING}) + + @classmethod + def stop_job(cls, job_id, stop_status): + schedule_logger(job_id).info(f"request stop job with {stop_status}") + jobs = ScheduleJobSaver.query_job(job_id=job_id) + if len(jobs) > 0: + if stop_status == JobStatus.CANCELED: + schedule_logger(job_id).info("cancel job") + set_cancel_status = schedule_utils.cancel_signal(job_id=job_id, set_or_reset=True) + schedule_logger(job_id).info(f"set job cancel signal {set_cancel_status}") + job = jobs[0] + job.f_status = stop_status + schedule_logger(job_id).info(f"request stop job with {stop_status} to all party") + status_code, response = FederatedScheduler.stop_job(job_id=job_id, roles=job.f_parties) + if status_code == FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job_id).info(f"stop job with {stop_status} successfully") + return ReturnCode.Base.SUCCESS, "success" + else: + tasks_group = ScheduleJobSaver.get_status_tasks_asc(job_id=job.f_job_id) + for task in tasks_group.values(): + TaskScheduler.collect_task_of_all_party(job, task=task, set_status=stop_status) + schedule_logger(job_id).info(f"stop job with {stop_status} failed, {response}") + return ReturnCode.Job.KILL_FAILED, json_dumps(response) + else: + raise NoFoundJob(job_id=job_id) + + @classmethod + def update_job_on_scheduler(cls, schedule_job: ScheduleJob, update_fields: list): + schedule_logger(schedule_job.f_job_id).info(f"try to update job {update_fields} on scheduler") + jobs = ScheduleJobSaver.query_job(job_id=schedule_job.f_job_id) + if not jobs: + raise Exception("Failed to update job status on scheduler") + job_info = schedule_job.to_human_model_dict(only_primary_with=update_fields) + for field in update_fields: + job_info[field] = getattr(schedule_job, "f_%s" % field) + if "status" in update_fields: + ScheduleJobSaver.update_job_status(job_info=job_info) + ScheduleJobSaver.update_job(job_info=job_info) + schedule_logger(schedule_job.f_job_id).info(f"update job {update_fields} on scheduler finished") + + @classmethod + def rerun_job(cls, job_id, auto, tasks: typing.List[ScheduleTaskStatus] = None): + schedule_logger(job_id).info(f"try to rerun job {job_id}") + jobs = ScheduleJobSaver.query_job(job_id=job_id) + if not jobs: + raise RuntimeError(f"can not found job {job_id}") + job = jobs[0] + if tasks: + schedule_logger(job_id).info(f"require {[task.f_task_name for task in tasks]} to rerun") + else: + # todo: get_need_revisit_nodes + tasks = ScheduleJobSaver.query_task(job_id=job_id, status=TaskStatus.CANCELED, scheduler_status=True) + job_can_rerun = any([TaskController.prepare_rerun_task( + job=job, task=task, auto=auto, force=False, + ) for task in tasks]) + schedule_logger(job_id).info("job set rerun signal") + status = schedule_utils.rerun_signal(job_id=job_id, set_or_reset=True) + schedule_logger(job_id).info(f"job set rerun signal {'successfully' if status else 'failed'}") + return True + + @classmethod + def finish(cls, job, end_status): + schedule_logger(job.f_job_id).info(f"job finished with {end_status}, do something...") + cls.stop_job(job_id=job.f_job_id, stop_status=end_status) + # todo: clean job + schedule_logger(job.f_job_id).info(f"job finished with {end_status}, done") + + +class TaskScheduler(object): + @classmethod + def schedule(cls, job): + schedule_logger(job.f_job_id).info("scheduling job tasks") + dag_schema = DAGSchema(**job.f_dag) + job_parser = FlowHub.load_job_parser(DAGSchema(**job.f_dag)) + tasks_group = ScheduleJobSaver.get_status_tasks_asc(job_id=job.f_job_id) + waiting_tasks = {} + auto_rerun_tasks = [] + job_interrupt = False + canceled = job.f_cancel_signal + for task in tasks_group.values(): + if task.f_sync_type == FederatedCommunicationType.POLL: + cls.collect_task_of_all_party(job=job, task=task) + else: + pass + new_task_status = cls.get_federated_task_status(job_id=task.f_job_id, task_id=task.f_task_id, + task_version=task.f_task_version) + task_interrupt = False + task_status_have_update = False + if new_task_status != task.f_status: + task_status_have_update = True + schedule_logger(job.f_job_id).info(f"sync task status {task.f_status} to {new_task_status}") + task.f_status = new_task_status + FederatedScheduler.sync_task_status(task_id=task.f_task_id, command_body={"status": task.f_status}) + ScheduleJobSaver.update_task_status(task.to_human_model_dict(), scheduler_status=True) + if InterruptStatus.contains(new_task_status): + task_interrupt = True + job_interrupt = True + if task.f_status == TaskStatus.WAITING: + waiting_tasks[task.f_task_name] = task + elif task_status_have_update and EndStatus.contains(task.f_status) or task_interrupt: + schedule_logger(task.f_job_id).info(f"stop task with status: {task.f_status}") + FederatedScheduler.stop_task(task_id=task.f_task_id, command_body={"status": task.f_status}) + if not canceled and AutoRerunStatus.contains(task.f_status): + if task.f_auto_retries > 0: + auto_rerun_tasks.append(task) + schedule_logger(job.f_job_id).info(f"task {task.f_task_id} {task.f_status} will be retried") + else: + schedule_logger(job.f_job_id).info(f"task {task.f_task_id} {task.f_status} has no retry count") + + scheduling_status_code = SchedulingStatusCode.NO_NEXT + schedule_logger(job.f_job_id).info(f"canceled status {canceled}, job interrupt status {job_interrupt}") + if not canceled and not job_interrupt: + for task_id, waiting_task in waiting_tasks.items(): + dependent_tasks = job_parser.infer_dependent_tasks( + dag_schema.dag.tasks[waiting_task.f_task_name].inputs + ) + schedule_logger(job.f_job_id).info(f"task {waiting_task.f_task_name} dependent tasks:{dependent_tasks}") + for task_name in dependent_tasks: + dependent_task = tasks_group[task_name] + if dependent_task.f_status != TaskStatus.SUCCESS: + break + else: + scheduling_status_code = SchedulingStatusCode.HAVE_NEXT + status_code = cls.start_task(job=job, task=waiting_task) + if status_code == SchedulingStatusCode.NO_RESOURCE: + schedule_logger(job.f_job_id).info( + f"task {waiting_task.f_task_id} can not apply resource, wait for the next round of scheduling") + break + elif status_code == SchedulingStatusCode.FAILED: + schedule_logger(job.f_job_id).info(f"task status code: {status_code}") + scheduling_status_code = SchedulingStatusCode.FAILED + waiting_task.f_status = StatusSet.FAILED + FederatedScheduler.sync_task_status(task_id=waiting_task.f_task_id, command_body={ + "status": waiting_task.f_status}) + break + else: + schedule_logger(job.f_job_id).info("have cancel signal, pass start job tasks") + schedule_logger(job.f_job_id).info("finish scheduling job tasks") + return scheduling_status_code, auto_rerun_tasks, tasks_group.values() + + @classmethod + def start_task(cls, job, task): + schedule_logger(task.f_job_id).info("try to start task {} {}".format(task.f_task_id, task.f_task_version)) + # apply resource for task + apply_status = cls.apply_task_resource(task, job) + if not apply_status: + return SchedulingStatusCode.NO_RESOURCE + task.f_status = TaskStatus.RUNNING + ScheduleJobSaver.update_task_status( + task_info=task.to_human_model_dict(only_primary_with=["status"]), scheduler_status=True + ) + schedule_logger(task.f_job_id).info("start task {} {}".format(task.f_task_id, task.f_task_version)) + FederatedScheduler.sync_task_status(task_id=task.f_task_id, command_body={"status": task.f_status}) + ScheduleJobSaver.update_task_status(task.to_human_model_dict(), scheduler_status=True) + status_code, response = FederatedScheduler.start_task(task_id=task.f_task_id) + if status_code == FederatedSchedulingStatusCode.SUCCESS: + return SchedulingStatusCode.SUCCESS + else: + return SchedulingStatusCode.FAILED + + @classmethod + def apply_task_resource(cls, task, job): + apply_status_code, federated_response = FederatedScheduler.resource_for_task( + task_id=task.f_task_id, + operation_type=ResourceOperation.APPLY.value + ) + if apply_status_code == FederatedSchedulingStatusCode.SUCCESS: + return True + else: + # rollback resource + rollback_party = [] + failed_party = [] + for dest_role in federated_response.keys(): + for dest_party_id in federated_response[dest_role].keys(): + retcode = federated_response[dest_role][dest_party_id]["code"] + if retcode == ReturnCode.Base.SUCCESS: + rollback_party.append({"role": dest_role, "party_id": [dest_party_id]}) + else: + failed_party.append({"role": dest_role, "party_id": [dest_party_id]}) + schedule_logger(job.f_job_id).info("task apply resource failed on {}, rollback {}".format(failed_party, + rollback_party)) + if rollback_party: + return_status_code, federated_response = FederatedScheduler.resource_for_task( + task_id=task.f_task_id, + roles=rollback_party, + operation_type=ResourceOperation.RETURN.value + ) + if return_status_code != FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job.f_job_id).info(f"task return resource failed:\n{federated_response}") + else: + schedule_logger(job.f_job_id).info("task no party should be rollback resource") + return False + + @classmethod + def collect_task_of_all_party(cls, job, task, set_status=None): + tasks_on_all_party = ScheduleJobSaver.query_task(task_id=task.f_task_id, task_version=task.f_task_version) + # tasks_status_on_all = set([task.f_status for task in tasks_on_all_party]) + # if not len(tasks_status_on_all) > 1 and TaskStatus.RUNNING not in tasks_status_on_all: + # return + status, federated_response = FederatedScheduler.collect_task(task_id=task.f_task_id) + if status != FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job.f_job_id).warning(f"collect task {task.f_task_id} {task.f_task_version} failed") + for _role in federated_response.keys(): + for _party_id, party_response in federated_response[_role].items(): + if party_response["code"] == ReturnCode.Base.SUCCESS: + schedule_logger(job.f_job_id).info( + f"collect party id {_party_id} task info: {party_response['data']}") + ScheduleJobSaver.update_task_status(task_info=party_response["data"]) + elif set_status: + tmp_task_info = { + "job_id": task.f_job_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version, + "role": _role, + "party_id": _party_id, + "party_status": set_status + } + ScheduleJobSaver.update_task_status(task_info=tmp_task_info) + + @classmethod + def get_federated_task_status(cls, job_id, task_id, task_version): + tasks_on_all_party = ScheduleJobSaver.query_task(task_id=task_id, task_version=task_version) + tasks_party_status = [task.f_status for task in tasks_on_all_party] + status = cls.calculate_multi_party_task_status(tasks_party_status) + schedule_logger(job_id=job_id).info( + "task {} {} status is {}, calculate by task party status list: {}".format(task_id, task_version, status, + tasks_party_status)) + return status + + @classmethod + def calculate_multi_party_task_status(cls, tasks_party_status): + tmp_status_set = set(tasks_party_status) + if TaskStatus.PASS in tmp_status_set: + tmp_status_set.remove(TaskStatus.PASS) + tmp_status_set.add(TaskStatus.SUCCESS) + if len(tmp_status_set) == 1: + return tmp_status_set.pop() + else: + for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), + reverse=False): + if status in tmp_status_set: + return status + if TaskStatus.RUNNING in tmp_status_set: + return TaskStatus.RUNNING + if TaskStatus.SUCCESS in tmp_status_set: + return TaskStatus.RUNNING + raise Exception("Calculate task status failed: {}".format(tasks_party_status)) diff --git a/python/fate_flow/manager/components/__init__.py b/python/fate_flow/manager/components/__init__.py new file mode 100644 index 000000000..878d3a9c5 --- /dev/null +++ b/python/fate_flow/manager/components/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/python/fate_flow/manager/components/base.py b/python/fate_flow/manager/components/base.py new file mode 100644 index 000000000..b7cc51056 --- /dev/null +++ b/python/fate_flow/manager/components/base.py @@ -0,0 +1,45 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.spec.dag import PartySpec, DAGSchema, DAGSpec, JobConfSpec, TaskConfSpec, TaskSpec, \ + PartyTaskSpec, PartyTaskRefSpec, RuntimeInputArtifacts +from fate_flow.manager.service.provider_manager import ProviderManager + + +class Base: + @staticmethod + def local_dag_schema(task_name, component_ref, parameters, inputs=None, provider=None, role=None, party_id=None): + if not provider: + provider = ProviderManager.get_fate_flow_provider() + if not role or not party_id: + role = "local" + party_id = "0" + party = PartySpec(role=role, party_id=[party_id]) + dag = DAGSchema( + schema_version=provider.version, + dag=DAGSpec( + conf=JobConfSpec(task=TaskConfSpec(provider=provider.provider_name)), + parties=[party], + stage="default", + tasks={task_name: TaskSpec(component_ref=component_ref, parties=[party])}, + party_tasks={ + f"{role}_{party_id}": PartyTaskSpec( + parties=[party], + tasks={task_name: PartyTaskRefSpec(parameters=parameters)} + )} + )) + if inputs: + dag.dag.tasks[task_name].inputs = RuntimeInputArtifacts(**inputs) + return dag diff --git a/python/fate_flow/manager/components/component_manager.py b/python/fate_flow/manager/components/component_manager.py new file mode 100644 index 000000000..c736adde7 --- /dev/null +++ b/python/fate_flow/manager/components/component_manager.py @@ -0,0 +1,94 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import uuid + +from fate_flow.controller.job_controller import JobController +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.types import EngineType +from fate_flow.manager.components.base import Base +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.runtime.system_settings import ENGINES, STORAGE +from fate_flow.engine import storage +from fate_flow.errors.server_error import ExistsTable + +class ComponentManager(Base): + @classmethod + def upload(cls, file, head, partitions, meta, namespace, name, extend_sid): + parameters = { + "file": file, + "head": head, + "partitions": partitions, + "meta": meta, + "extend_sid": extend_sid + } + if not name or not namespace: + name = str(uuid.uuid1()) + namespace = "upload" + parameters.update({ + "storage_engine": ENGINES.get(EngineType.STORAGE), + "name": name, + "namespace": namespace + }) + address = STORAGE.get(ENGINES.get(EngineType.STORAGE)) + if address: + parameters.update({"address": address}) + dag_schema = cls.local_dag_schema( + task_name="upload_0", + component_ref="upload", + parameters=parameters + ) + result = JobController.request_create_job(dag_schema.dict(), is_local=True) + if result.get("code") == ReturnCode.Base.SUCCESS: + result["data"] = {"name": name, "namespace": namespace} + return result + + @classmethod + def dataframe_transformer(cls, data_warehouse, namespace, name, drop, site_name): + data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) + if data_table_meta: + if not drop: + raise ExistsTable( + name=name, + namespace=namespace, + warning="If you want to ignore this error and continue transformer, " + "you can set the parameter of 'drop' to 'true' " + ) + data_table_meta.destroy_metas() + provider = ProviderManager.get_default_fate_provider() + dag_schema = cls.local_dag_schema( + task_name="transformer_0", + component_ref="dataframe_transformer", + parameters={"namespace": namespace, "name": name, "site_name": site_name}, + inputs={"data": {"table": {"data_warehouse": data_warehouse}}}, + provider=provider + ) + result = JobController.request_create_job(dag_schema.dict(), is_local=True) + if result.get("code") == ReturnCode.Base.SUCCESS: + result["data"] = {"name": name, "namespace": namespace} + return result + + @classmethod + def download(cls, namespace, name, path): + dag_schema = cls.local_dag_schema( + task_name="download_0", + component_ref="download", + parameters=dict(namespace=namespace, name=name, path=path) + ) + result = JobController.request_create_job(dag_schema.dict(), is_local=True) + if result.get("code") == ReturnCode.Base.SUCCESS: + result["data"] = {"name": name, "namespace": namespace, "path": path} + return result + diff --git a/python/fate_flow/manager/components/download.py b/python/fate_flow/manager/components/download.py new file mode 100644 index 000000000..08cc0d809 --- /dev/null +++ b/python/fate_flow/manager/components/download.py @@ -0,0 +1,59 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging as logger +import os + +from fate_flow.engine import storage +from fate_flow.manager.data.data_manager import DataManager + + +class Param(object): + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + if v is None: + continue + d[k] = v + return d + + +class DownloadParam(Param): + def __init__( + self, + dir_name="", + namespace="", + name="" + ): + self.dir_name = dir_name + self.namespace = namespace + self.name = name + + +class Download: + def __init__(self): + self.table = None + self.schema = {} + + def run(self, parameters: DownloadParam, job_id=""): + data_table_meta = storage.StorageTableMeta( + name=parameters.name, + namespace=parameters.namespace + ) + DataManager.send_table( + output_tables_meta={"table": data_table_meta}, + download_dir=os.path.abspath(parameters.dir_name) + ) + diff --git a/python/fate_flow/manager/container/__init__.py b/python/fate_flow/manager/container/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/manager/container/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/manager/docker_manager.py b/python/fate_flow/manager/container/docker_manager.py similarity index 52% rename from python/fate_flow/manager/docker_manager.py rename to python/fate_flow/manager/container/docker_manager.py index 55e5e4f2f..3161fc0cb 100644 --- a/python/fate_flow/manager/docker_manager.py +++ b/python/fate_flow/manager/container/docker_manager.py @@ -14,37 +14,36 @@ # limitations under the License. import docker -from fate_flow.settings import LOG_DIRECTORY, WORKER, LOCAL_DATA_STORE_PATH +from fate_flow.runtime.component_provider import ComponentProvider class DockerManager: - config = WORKER.get('docker', {}).get('config', {}) - image = WORKER.get('docker', {}).get('image', '') - fate_root_dir = WORKER.get('docker', {}).get('fate_root_dir', '') - eggroll_conf_dir = WORKER.get('docker', {}).get('eggroll_conf_dir', '') + def __init__(self, provider: ComponentProvider): + self.provider = provider + self.client = docker.DockerClient(base_url=provider.metadata.base_url) - def __init__(self): - self.client = docker.DockerClient(**self.config) - - def start(self, name, command, environment): + def start(self, name, command, environment, auto_remove=False, detach=True, network_mode="host"): + # todo: delete volumes + # volumes = { + # LOG_DIRECTORY: { + # 'bind': LOG_DIRECTORY, + # 'mode': 'rw', + # }, + # eggroll_conf_dir: { + # 'bind': self.provider.metadata.eggroll_conf_dir, + # 'mode': 'ro', + # }, + # LOCAL_DATA_STORE_PATH: { + # 'bind': LOCAL_DATA_STORE_PATH, + # 'mode': 'rw', + # } + # } + volumes = {} self.client.containers.run( - self.image, command, - auto_remove=False, detach=True, + self.provider.metadata.image, command, + auto_remove=auto_remove, detach=detach, environment=environment, name=name, - network_mode='host', volumes={ - LOG_DIRECTORY: { - 'bind': LOG_DIRECTORY, - 'mode': 'rw', - }, - self.eggroll_conf_dir: { - 'bind': f'{self.fate_root_dir}/eggroll/conf', - 'mode': 'ro', - }, - LOCAL_DATA_STORE_PATH: { - 'bind': LOCAL_DATA_STORE_PATH, - 'mode': 'rw', - } - }, + network_mode=network_mode, volumes=volumes ) def stop(self, name): diff --git a/python/fate_flow/manager/k8s_conf_template.yaml b/python/fate_flow/manager/container/k8s_conf_template.yaml similarity index 100% rename from python/fate_flow/manager/k8s_conf_template.yaml rename to python/fate_flow/manager/container/k8s_conf_template.yaml diff --git a/python/fate_flow/manager/k8s_manager.py b/python/fate_flow/manager/container/k8s_manager.py similarity index 95% rename from python/fate_flow/manager/k8s_manager.py rename to python/fate_flow/manager/container/k8s_manager.py index a340c4879..9ef001cd6 100644 --- a/python/fate_flow/manager/k8s_manager.py +++ b/python/fate_flow/manager/container/k8s_manager.py @@ -18,7 +18,8 @@ from kubernetes import client, config from ruamel import yaml -from fate_flow.settings import WORKER +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.runtime.system_settings import WORKER from fate_flow.utils.conf_utils import get_base_config from fate_flow.utils.log import getLogger @@ -28,7 +29,8 @@ class K8sManager: image = WORKER.get('k8s', {}).get('image', '') namespace = WORKER.get('k8s', {}).get('namespace', '') - def __init__(self): + + def __init__(self, provider: ComponentProvider): config.load_kube_config() self.job_template = yaml.safe_load( (Path(__file__).parent / 'k8s_template.yaml').read_text('utf-8') diff --git a/python/fate_flow/manager/k8s_template.yaml b/python/fate_flow/manager/container/k8s_template.yaml similarity index 100% rename from python/fate_flow/manager/k8s_template.yaml rename to python/fate_flow/manager/container/k8s_template.yaml diff --git a/python/fate_flow/manager/data/__init__.py b/python/fate_flow/manager/data/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/manager/data/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/manager/data/data_manager.py b/python/fate_flow/manager/data/data_manager.py new file mode 100644 index 000000000..5e2cd2a99 --- /dev/null +++ b/python/fate_flow/manager/data/data_manager.py @@ -0,0 +1,312 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +import pickle +import tarfile +import uuid +from tempfile import TemporaryDirectory + +from flask import send_file + +from fate_flow.engine import storage +from fate_flow.engine.storage import Session, StorageEngine, DataType +from fate_flow.entity.types import EggRollAddress, StandaloneAddress, HDFSAddress, PathAddress, ApiAddress +from fate_flow.errors.server_error import NoFoundTable +from fate_flow.manager.service.output_manager import OutputDataTracking +from fate_flow.runtime.system_settings import LOCALFS_DATA_HOME, STANDALONE_DATA_HOME, STORAGE +from fate_flow.utils import job_utils +from fate_flow.utils.io_utils import URI + +DELIMITER = '\t' + + +class DataManager: + @classmethod + def send_table( + cls, + output_tables_meta, + tar_file_name="", + need_head=True, + download_dir="", + + ): + if not need_head: + need_head = True + output_data_file_list = [] + output_data_meta_file_list = [] + with TemporaryDirectory() as output_tmp_dir: + for output_name, output_table_metas in output_tables_meta.items(): + if not isinstance(output_table_metas, list): + output_table_metas = [output_table_metas] + for index, output_table_meta in enumerate(output_table_metas): + if not download_dir: + output_data_file_path = "{}/{}/{}.csv".format(output_tmp_dir, output_name, index) + output_data_meta_file_path = "{}/{}/{}.meta".format(output_tmp_dir, output_name, index) + else: + output_data_file_path = "{}/{}/{}.csv".format(download_dir, output_name, index) + output_data_meta_file_path = "{}/{}/{}.meta".format(download_dir, output_name, index) + output_data_file_list.append(output_data_file_path) + output_data_meta_file_list.append(output_data_meta_file_path) + os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True) + with Session() as sess: + if not output_table_meta: + raise NoFoundTable() + table = sess.get_table( + name=output_table_meta.get_name(), + namespace=output_table_meta.get_namespace()) + cls.write_data_to_file(output_data_file_path, output_data_meta_file_path, table, need_head) + if download_dir: + return + # tar + output_data_tarfile = "{}/{}".format(output_tmp_dir, tar_file_name) + tar = tarfile.open(output_data_tarfile, mode='w:gz') + for index in range(0, len(output_data_file_list)): + tar.add(output_data_file_list[index], os.path.relpath(output_data_file_list[index], output_tmp_dir)) + tar.add(output_data_meta_file_list[index], + os.path.relpath(output_data_meta_file_list[index], output_tmp_dir)) + tar.close() + return send_file(output_data_tarfile, download_name=tar_file_name, as_attachment=True, mimetype='application/gzip') + + @classmethod + def write_data_to_file(cls, output_data_file_path, output_data_meta_file_path, table, need_head): + with open(output_data_file_path, 'w') as fw: + data_meta = table.meta.get_data_meta() + header = cls.get_data_header(table.meta.get_id_delimiter(), data_meta) + with open(output_data_meta_file_path, 'w') as f: + json.dump({'header': header}, f, indent=4) + if table: + write_header = False + for v in cls.collect_data(table=table): + # save meta + if not write_header and need_head and header and table.meta.get_have_head(): + if isinstance(header, list): + header = table.meta.get_id_delimiter().join(header) + fw.write(f'{header}\n') + write_header = True + delimiter = table.meta.get_id_delimiter() + if isinstance(v, str): + fw.write('{}\n'.format(v)) + elif isinstance(v, list): + fw.write('{}\n'.format(delimiter.join([str(_v) for _v in v]))) + else: + raise ValueError(type(v)) + + @staticmethod + def collect_data(table): + if table.data_type == DataType.DATAFRAME: + for _, data in table.collect(): + for v in data: + yield v + elif table.data_type == DataType.TABLE: + for _k, _v in table.collect(): + yield table.meta.get_id_delimiter().join([_k, _v]) + else: + return [] + + @staticmethod + def display_data(table_metas): + datas = {} + for key, metas in table_metas.items(): + datas[key] = [] + for meta in metas: + if meta.data_type in [DataType.DATAFRAME, DataType.TABLE]: + datas[key].append({"data": meta.get_part_of_data(), "metadata": meta.get_data_meta()}) + else: + continue + return datas + + @classmethod + def query_output_data_table(cls, **kwargs): + data_list = OutputDataTracking.query(**kwargs) + outputs = {} + for data in data_list: + if data.f_output_key not in outputs: + outputs[data.f_output_key] = [] + outputs[data.f_output_key].append({"namespace": data.f_namespace, "name": data.f_name}) + return outputs + + @classmethod + def download_output_data(cls, tar_file_name, **kwargs): + outputs = {} + for key, tables in cls.query_output_data_table(**kwargs).items(): + if key not in outputs: + outputs[key] = [] + for table in tables: + outputs[key].append(storage.StorageTableMeta( + name=table.get("name"), + namespace=table.get("namespace") + )) + + if not outputs: + raise NoFoundTable() + + return cls.send_table(outputs, tar_file_name=tar_file_name) + + @classmethod + def display_output_data(cls, **kwargs): + outputs = {} + for key, tables in cls.query_output_data_table(**kwargs).items(): + if key not in outputs: + outputs[key] = [] + for table in tables: + outputs[key].append(storage.StorageTableMeta( + name=table.get("name"), + namespace=table.get("namespace") + )) + return cls.display_data(outputs) + + @staticmethod + def delete_data(namespace, name): + with Session() as sess: + table = sess.get_table(name=name, namespace=namespace) + if table: + table.destroy() + return True + return False + + @staticmethod + def create_data_table( + namespace, name, uri, partitions, data_meta, data_type, part_of_data=None, count=None, source=None + ): + engine, address = DataManager.uri_to_address(uri) + storage_meta = storage.StorageTableBase( + namespace=namespace, name=name, address=address, + partitions=partitions, engine=engine, + options=None + ) + storage_meta.create_meta( + data_meta=data_meta, part_of_data=part_of_data, count=count, source=source, data_type=data_type + ) + + @staticmethod + def uri_to_address(uri): + uri_schema = URI.from_string(uri).to_schema() + engine = uri_schema.schema() + if engine == StorageEngine.EGGROLL: + address = EggRollAddress(namespace=uri_schema.namespace, name=uri_schema.name) + elif uri_schema.schema() == StorageEngine.STANDALONE: + address = StandaloneAddress(namespace=uri_schema.namespace, name=uri_schema.name) + elif uri_schema.schema() == StorageEngine.HDFS: + address = HDFSAddress(path=uri_schema.path, name_node=uri_schema.authority) + elif uri_schema.schema() in [StorageEngine.PATH, StorageEngine.FILE]: + address = PathAddress(path=uri_schema.path) + elif uri_schema.schema() in [StorageEngine.HTTP]: + address = ApiAddress(url=uri_schema.path) + else: + raise ValueError(f"uri {uri} engine could not be converted to an address") + return engine, address + + @staticmethod + def get_data_info(namespace, name): + data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) + if data_table_meta: + data = { + "namespace": namespace, + "name": name, + "count": data_table_meta.count, + "meta": data_table_meta.get_data_meta(), + "engine": data_table_meta.engine, + "path": data_table_meta.address.engine_path, + "source": data_table_meta.source, + "data_type": data_table_meta.data_type + } + display_data = data_table_meta.part_of_data + return data, display_data + raise NoFoundTable(name=name, namespace=namespace) + + @staticmethod + def get_data_header(delimiter, data_meta): + header = [] + if data_meta.get("header"): + header = data_meta.get("header") + if isinstance(header, str): + header = header.split(delimiter) + else: + for field in data_meta.get("schema_meta", {}).get("fields", []): + header.append(field.get("name")) + return header + + @staticmethod + def deserialize_data(m): + fields = m.partition(DELIMITER) + return fields[0], pickle.loads(bytes.fromhex(fields[2])) + + @staticmethod + def serialize_data(k, v): + return f"{k}{DELIMITER}{pickle.dumps(v).hex()}" + + +class DatasetManager: + @staticmethod + def task_output_name(task_id, task_version): + return f"output_data_{task_id}_{task_version}", uuid.uuid1().hex + + @staticmethod + def get_output_name(uri): + namespace, name = uri.split("/")[-2], uri.split("/")[-1] + return namespace, name + + @classmethod + def upload_data_path(cls, name, namespace, prefix=None, storage_engine=StorageEngine.HDFS): + if storage_engine == StorageEngine.HDFS: + return cls.default_hdfs_path(data_type="input", name=name, namespace=namespace, prefix=prefix) + elif storage_engine == StorageEngine.FILE: + return cls.default_localfs_path(data_type="input", name=name, namespace=namespace) + + @classmethod + def output_data_uri(cls, storage_engine, task_id, is_multi=False): + if storage_engine == StorageEngine.STANDALONE: + uri = f"{storage_engine}://{STANDALONE_DATA_HOME}/{task_id}/{uuid.uuid1().hex}" + elif storage_engine == StorageEngine.HDFS: + uri = cls.default_output_fs_path(uuid.uuid1().hex, task_id, storage_engine=storage_engine) + elif storage_engine == StorageEngine.FILE: + uri = f"file://{cls.default_output_fs_path(uuid.uuid1().hex, task_id, storage_engine=storage_engine)}" + else: + # egg: eggroll + uri = f"{storage_engine}:///{task_id}/{uuid.uuid1().hex}" + + if is_multi: + # replace "{index}" + uri += "_{index}" + return uri + + @classmethod + def output_local_uri(cls, name, type_name, task_info, is_multi=False): + path = job_utils.get_task_directory(**task_info, output=True) + uri = os.path.join(f"file://{path}", name, type_name) + if is_multi: + # replace "{index}" + uri += "_{index}" + return uri + + @classmethod + def default_output_fs_path(cls, name, namespace, prefix=None, storage_engine=StorageEngine.HDFS): + if storage_engine == StorageEngine.HDFS: + return f'{STORAGE.get(storage_engine).get("name_node")}' \ + f'{cls.default_hdfs_path(data_type="output", name=name, namespace=namespace, prefix=prefix)}' + elif storage_engine == StorageEngine.FILE: + return cls.default_localfs_path(data_type="output", name=name, namespace=namespace) + + @staticmethod + def default_localfs_path(name, namespace, data_type): + return os.path.join(LOCALFS_DATA_HOME, namespace, name) + + @staticmethod + def default_hdfs_path(data_type, name, namespace, prefix=None): + p = f"/fate/{data_type}/{namespace}/{name}" + if prefix: + p = f"{prefix}/{p}" + return p diff --git a/python/fate_flow/manager/data_manager.py b/python/fate_flow/manager/data_manager.py deleted file mode 100644 index 77df777e7..000000000 --- a/python/fate_flow/manager/data_manager.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import datetime -import io -import json -import os -import tarfile - -from flask import send_file - -from fate_flow.engine.storage import Session -from fate_flow.settings import stat_logger -from fate_flow.utils.file_utils import get_fate_flow_directory - - -class DataManager: - @staticmethod - def send_table( - output_tables_meta, - tar_file_name="", - limit=-1, - need_head=True, - local_download=False, - output_data_file_path=None - ): - output_data_file_list = [] - output_data_meta_file_list = [] - output_tmp_dir = os.path.join(get_fate_flow_directory(), 'tmp/{}/{}'.format(datetime.datetime.now().strftime("%Y%m%d"), fate_uuid())) - for output_name, output_table_meta in output_tables_meta.items(): - output_data_count = 0 - if not local_download: - output_data_file_path = "{}/{}.csv".format(output_tmp_dir, output_name) - output_data_meta_file_path = "{}/{}.meta".format(output_tmp_dir, output_name) - os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True) - with open(output_data_file_path, 'w') as fw: - with Session() as sess: - output_table = sess.get_table(name=output_table_meta.get_name(), - namespace=output_table_meta.get_namespace()) - all_extend_header = {} - if output_table: - for k, v in output_table.collect(): - # save meta - if output_data_count == 0: - output_data_file_list.append(output_data_file_path) - header = output_table_meta.get_id_delimiter.join([ - output_table_meta.get_schema().get("sid"), - output_table_meta.get_schema().get("header")] - ) - - if not local_download: - output_data_meta_file_list.append(output_data_meta_file_path) - with open(output_data_meta_file_path, 'w') as f: - json.dump({'header': header}, f, indent=4) - if need_head and header and output_table_meta.get_have_head(): - fw.write('{}\n'.format(','.join(header))) - delimiter = output_table_meta.get_id_delimiter() if output_table_meta.get_id_delimiter() else "," - fw.write('{}\n'.format(delimiter.join(map(lambda x: str(x), data_line)))) - output_data_count += 1 - if output_data_count == limit: - break - if local_download: - return - # tar - output_data_tarfile = "{}/{}".format(output_tmp_dir, tar_file_name) - tar = tarfile.open(output_data_tarfile, mode='w:gz') - for index in range(0, len(output_data_file_list)): - tar.add(output_data_file_list[index], os.path.relpath(output_data_file_list[index], output_tmp_dir)) - tar.add(output_data_meta_file_list[index], - os.path.relpath(output_data_meta_file_list[index], output_tmp_dir)) - tar.close() - for key, path in enumerate(output_data_file_list): - try: - os.remove(path) - os.remove(output_data_meta_file_list[key]) - except Exception as e: - # warning - stat_logger.warning(e) - return send_file(output_data_tarfile, attachment_filename=tar_file_name, as_attachment=True) \ No newline at end of file diff --git a/python/fate_flow/manager/log/__init__.py b/python/fate_flow/manager/log/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/manager/log/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/manager/log/log_manager.py b/python/fate_flow/manager/log/log_manager.py new file mode 100644 index 000000000..235437411 --- /dev/null +++ b/python/fate_flow/manager/log/log_manager.py @@ -0,0 +1,88 @@ +import os +import subprocess + +from fate_flow.runtime.system_settings import LOG_DIR +from fate_flow.utils.log_utils import replace_ip + +JOB = ["schedule_info", "schedule_error"] +TASK = ["task_error", "task_info", "task_warning", "task_debug"] + + +def parameters_check(log_type, job_id, role, party_id, task_name): + if log_type in JOB: + if not job_id: + return False + if log_type in TASK: + if not job_id or not role or not party_id or not task_name: + return False + return True + + +class LogManager: + def __init__(self, log_type, job_id, party_id="", role="", task_name="", **kwargs): + self.log_type = log_type + self.job_id = job_id + self.party_id = party_id + self.role = role + self.task_name = task_name + + @property + def task_base_path(self): + if self.role and self.party_id and self.task_name: + return os.path.join(self.job_id, self.role, self.party_id, self.task_name, "root") + else: + return "" + + @property + def file_path(self): + status = parameters_check(self.log_type, self.job_id, self.role, self.party_id, self.task_name) + if not status: + raise Exception(f"job type {self.log_type} Missing parameters") + type_dict = { + "schedule_info": os.path.join(self.job_id, "fate_flow_schedule.log"), + "schedule_error": os.path.join(self.job_id, "fate_flow_schedule_error.log"), + "task_error": os.path.join(self.task_base_path, "ERROR"), + "task_warning": os.path.join(self.task_base_path, "WARNING"), + "task_info": os.path.join(self.task_base_path, "INFO"), + "task_debug": os.path.join(self.task_base_path, "DEBUG") + } + if self.log_type not in type_dict.keys(): + raise Exception(f"no found log type {self.log_type}") + return os.path.join(LOG_DIR, type_dict[self.log_type]) + + def cat_log(self, begin, end): + line_list = [] + log_path = self.file_path + if begin and end: + cmd = f"cat {log_path} | tail -n +{begin}| head -n {end-begin+1}" + elif begin: + cmd = f"cat {log_path} | tail -n +{begin}" + elif end: + cmd = f"cat {log_path} | head -n {end}" + else: + cmd = f"cat {log_path}" + lines = self.execute(cmd) + if lines: + line_list = [] + line_num = begin if begin else 1 + for line in lines.split("\n"): + line = replace_ip(line) + line_list.append({"line_num": line_num, "content": line}) + line_num += 1 + return line_list + + def count(self): + try: + if os.path.exists(self.file_path): + return int(self.execute(f"cat {self.file_path} | wc -l").strip()) + return 0 + except: + return 0 + + @staticmethod + def execute(cmd): + res = subprocess.run( + cmd, shell=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) + return res.stdout diff --git a/python/fate_flow/manager/metric/__init__.py b/python/fate_flow/manager/metric/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/manager/metric/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/manager/output_manager.py b/python/fate_flow/manager/metric/metric_manager.py similarity index 52% rename from python/fate_flow/manager/output_manager.py rename to python/fate_flow/manager/metric/metric_manager.py index bf18f8b46..b2bc61d03 100644 --- a/python/fate_flow/manager/output_manager.py +++ b/python/fate_flow/manager/metric/metric_manager.py @@ -13,36 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import operator +from typing import List -from fate_flow.db.base_models import DB, BaseModelOperate -from fate_flow.db.db_models import TrackingOutputInfo, Metric -from fate_flow.entity.output_types import MetricData +from fate_flow.db.base_models import DB +from fate_flow.db.db_models import Metric +from fate_flow.entity.spec.dag import MetricData from fate_flow.utils import db_utils from fate_flow.utils.log_utils import schedule_logger -class OutputDataTracking(BaseModelOperate): - @classmethod - def create(cls, entity_info): - # name, namespace, key, meta, job_id, role, party_id, task_id, task_version - cls._create_entity(TrackingOutputInfo, entity_info) - - @classmethod - def query(cls, reverse=False, **kwargs): - return cls._query(TrackingOutputInfo, reverse=reverse, **kwargs) - - -class OutputModel(BaseModelOperate): - @classmethod - def create(cls, entity_info): - # name, namespace, key, meta, job_id, role, party_id, task_id, task_version - cls._create_entity(TrackingOutputInfo, entity_info) - - @classmethod - def query(cls, reverse=False, **kwargs): - return cls._query(TrackingOutputInfo, reverse=reverse, **kwargs) - - class OutputMetric: def __init__(self, job_id: str, role: str, party_id: str, task_name: str, task_id: str = None, task_version: int = None): @@ -53,46 +32,50 @@ def __init__(self, job_id: str, role: str, party_id: str, task_name: str, task_i self.task_id = task_id self.task_version = task_version - def save_output_metrics(self, data, incomplete): - return self._insert_metrics_into_db(MetricData(**data), incomplete) + def save_output_metrics(self, data): + if not data or not isinstance(data, list): + raise RuntimeError(f"Save metric data failed, data is {data}") + return self._insert_metrics_into_db( + self.job_id, self.role, self.party_id, self.task_id, self.task_version, self.task_name, data + ) - @DB.connection_context() - def _insert_metrics_into_db(self, data: MetricData, incomplete: bool): - try: - model_class = self.get_model_class() - if not model_class.table_exists(): - model_class.create_table() - tracking_metric = model_class() - tracking_metric.f_job_id = self.job_id - tracking_metric.f_task_id = self.task_id - tracking_metric.f_task_version = self.task_version - tracking_metric.f_role = self.role - tracking_metric.f_party_id = self.party_id - tracking_metric.f_task_name = self.task_name + def save_as(self, job_id, role, party_id, task_name, task_id, task_version): + data_list = self.read_metrics() + self._insert_metrics_into_db( + job_id, role, party_id, task_id, task_version, task_name, data_list + ) - tracking_metric.f_namespace = data.namespace - tracking_metric.f_name = data.name - tracking_metric.f_type = data.type - tracking_metric.f_groups = data.groups - tracking_metric.f_metadata = data.metadata - tracking_metric.f_data = data.data - tracking_metric.f_incomplete = incomplete - tracking_metric.save() - except Exception as e: - schedule_logger(self.job_id).exception( - "An exception where inserted metric {} of metric namespace: {} to database:\n{}".format( - data.name, - data.namespace, - e - )) + @DB.connection_context() + def _insert_metrics_into_db(self, job_id, role, party_id, task_id, task_version, task_name, data_list): + model_class = self.get_model_class(job_id) + if not model_class.table_exists(): + model_class.create_table() + metric_list = [{ + "f_job_id": job_id, + "f_task_id": task_id, + "f_task_version": task_version, + "f_role": role, + "f_party_id": party_id, + "f_task_name": task_name, + "f_name": data.get("name"), + "f_type": data.get("type"), + "f_groups": data.get("groups"), + "f_step_axis": data.get("step_axis"), + "f_data": data.get("data") + + } for data in data_list] + + with DB.atomic(): + for i in range(0, len(metric_list), 100): + model_class.insert_many(metric_list[i: i+100]).execute() @DB.connection_context() def read_metrics(self, filters_args: dict = None): try: if not filters_args: filters_args = {} - tracking_metric_model = self.get_model_class() - key_list = ["namespace", "name", "type", "groups", "incomplete"] + tracking_metric_model = self.get_model_class(self.job_id) + key_list = ["name", "type", "groups", "step_axis"] filters = [ tracking_metric_model.f_job_id == self.job_id, tracking_metric_model.f_role == self.role, @@ -105,7 +88,11 @@ def read_metrics(self, filters_args: dict = None): if v is not None: filters.append(operator.attrgetter(f"f_{k}")(tracking_metric_model) == v) metrics = tracking_metric_model.select( - tracking_metric_model.f_data, tracking_metric_model.f_metadata + tracking_metric_model.f_name, + tracking_metric_model.f_type, + tracking_metric_model.f_groups, + tracking_metric_model.f_step_axis, + tracking_metric_model.f_data ).where(*filters) return [metric.to_human_model_dict() for metric in metrics] except Exception as e: @@ -115,13 +102,12 @@ def read_metrics(self, filters_args: dict = None): @DB.connection_context() def query_metric_keys(self): try: - tracking_metric_model = self.get_model_class() + tracking_metric_model = self.get_model_class(self.job_id) metrics = tracking_metric_model.select( - tracking_metric_model.f_namespace, tracking_metric_model.f_name, tracking_metric_model.f_type, tracking_metric_model.f_groups, - tracking_metric_model.f_incomplete + tracking_metric_model.f_step_axis ).where( tracking_metric_model.f_job_id == self.job_id, tracking_metric_model.f_role == self.role, @@ -135,15 +121,15 @@ def query_metric_keys(self): raise e @DB.connection_context() - def clean_metrics(self): - tracking_metric_model = self.get_model_class() + def delete_metrics(self): + tracking_metric_model = self.get_model_class(self.job_id) operate = tracking_metric_model.delete().where( tracking_metric_model.f_task_id == self.task_id, - tracking_metric_model.f_task_version == self.task_version, tracking_metric_model.f_role == self.role, tracking_metric_model.f_party_id == self.party_id ) return operate.execute() > 0 - def get_model_class(self): - return db_utils.get_dynamic_db_model(Metric, self.job_id) + @staticmethod + def get_model_class(job_id): + return db_utils.get_dynamic_db_model(Metric, job_id) diff --git a/python/fate_flow/manager/model/__init__.py b/python/fate_flow/manager/model/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/manager/model/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/manager/model/engine/__init__.py b/python/fate_flow/manager/model/engine/__init__.py new file mode 100644 index 000000000..67e7c8c82 --- /dev/null +++ b/python/fate_flow/manager/model/engine/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.manager.model.engine._tencent_cos import TencentCosStorage +from fate_flow.manager.model.engine._mysql import MysqlModelStorage + +__all__ = ["MysqlModelStorage", "TencentCosStorage"] diff --git a/python/fate_flow/manager/model/engine/_mysql.py b/python/fate_flow/manager/model/engine/_mysql.py new file mode 100644 index 000000000..a7539c2fe --- /dev/null +++ b/python/fate_flow/manager/model/engine/_mysql.py @@ -0,0 +1,141 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import io + +from copy import deepcopy + +from peewee import PeeweeException, CharField, IntegerField, CompositeKey +from playhouse.pool import PooledMySQLDatabase + +from fate_flow.db.base_models import LOGGER, BaseModel, LongTextField +from fate_flow.utils.password_utils import decrypt_database_config + +DB = PooledMySQLDatabase(None) +SLICE_MAX_SIZE = 1024 * 1024 * 8 + + +class MachineLearningModel(BaseModel): + f_storage_key = CharField(max_length=100) + f_content = LongTextField(default='') + f_slice_index = IntegerField(default=0) + + class Meta: + database = DB + db_table = 't_machine_learning_model' + primary_key = CompositeKey('f_storage_key', 'f_slice_index') + + +class MysqlModelStorage(object): + def __init__(self, storage_address, decrypt_key=None): + self.init_db(storage_address, decrypt_key) + + def exists(self, storage_key: str): + try: + with DB.connection_context(): + counts = MachineLearningModel.select().where( + MachineLearningModel.f_storage_key == storage_key + ).count() + return counts > 0 + except PeeweeException as e: + # Table doesn't exist + if e.args and e.args[0] == 1146: + return False + + raise e + finally: + self.close_connection() + + def delete(self, storage_key): + if not self.exists(storage_key): + raise FileNotFoundError(f'The model {storage_key} not found in the database.') + return MachineLearningModel.delete().where( + MachineLearningModel.f_storage_key == storage_key + ).execute() + + def store(self, memory_io, storage_key, force_update=True): + memory_io.seek(0) + if not force_update and self.exists(storage_key): + raise FileExistsError(f'The model {storage_key} already exists in the database.') + + try: + DB.create_tables([MachineLearningModel]) + with DB.connection_context(): + MachineLearningModel.delete().where( + MachineLearningModel.f_storage_key == storage_key + ).execute() + + LOGGER.info(f'Starting store model {storage_key}.') + + slice_index = 0 + while True: + content = memory_io.read(SLICE_MAX_SIZE) + if not content: + break + model_in_table = MachineLearningModel() + model_in_table.f_storage_key = storage_key + model_in_table.f_content = base64.b64encode(content) + model_in_table.f_slice_index = slice_index + + rows = model_in_table.save(force_insert=True) + if not rows: + raise IndexError(f'Save slice index {slice_index} failed') + + LOGGER.info(f'Saved slice index {slice_index} of model {storage_key}.') + slice_index += 1 + except Exception as e: + LOGGER.exception(e) + raise Exception(f'Store model {storage_key} to mysql failed.') + else: + LOGGER.info(f'Store model {storage_key} to mysql successfully.') + finally: + self.close_connection() + + def read(self, storage_key): + _io = io.BytesIO() + if not self.exists(storage_key): + raise Exception(f'model {storage_key} not exist in the database.') + try: + with DB.connection_context(): + models_in_tables = MachineLearningModel.select().where( + MachineLearningModel.f_storage_key == storage_key + ).order_by(MachineLearningModel.f_slice_index) + + for model in models_in_tables: + _io.write(base64.b64decode(model.f_content)) + + except Exception as e: + LOGGER.exception(e) + raise Exception(f'read model {storage_key} from mysql failed.') + else: + LOGGER.debug(f'read model from mysql successfully.') + finally: + self.close_connection() + return _io + + @staticmethod + def init_db(storage_address, decrypt_key): + _storage_address = deepcopy(storage_address) + database = _storage_address.pop('name') + decrypt_database_config(_storage_address, decrypt_key=decrypt_key) + DB.init(database, **_storage_address) + + @staticmethod + def close_connection(): + if DB: + try: + DB.close() + except Exception as e: + LOGGER.exception(e) diff --git a/python/fate_flow/manager/model/engine/_tencent_cos.py b/python/fate_flow/manager/model/engine/_tencent_cos.py new file mode 100644 index 000000000..e0a2bb0ab --- /dev/null +++ b/python/fate_flow/manager/model/engine/_tencent_cos.py @@ -0,0 +1,88 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io + +from copy import deepcopy + +from fate_flow.db.base_models import LOGGER + + +class TencentCosStorage(object): + def __init__(self, storage_address, decrypt_key=None): + self.Bucket = storage_address.get("Bucket") + self.client = self.init_client(storage_address) + + def exists(self, storage_key: str): + from qcloud_cos import CosServiceError + + try: + self.client.head_object( + Bucket=self.Bucket, + Key=storage_key, + ) + except CosServiceError as e: + if e.get_error_code() != 'NoSuchResource': + raise e + return False + else: + return True + + def store(self, memory_io, storage_key, force_update=True): + memory_io.seek(0) + if not force_update and self.exists(storage_key): + raise FileExistsError(f'The model {storage_key} already exists in the Cos.') + + try: + rt = self.client.put_object(Bucket=self.Bucket, Key=storage_key, Body=memory_io) + except Exception as e: + raise Exception(f"Store model {storage_key} to Tencent COS failed: {e}") + else: + LOGGER.info(f"Store model {storage_key} to Tencent COS successfully. " + f"ETag: {rt['ETag']}") + + def read(self, storage_key): + _io = io.BytesIO() + try: + rt = self.client.get_object( + Bucket=self.Bucket, + Key=storage_key + ) + _io.write(rt.get("Body").get_raw_stream().read()) + except Exception as e: + LOGGER.exception(e) + raise Exception(f"Read model {storage_key} from Tencent COS failed: {e}") + else: + LOGGER.info(f"Read model {storage_key} from Tencent COS successfully") + return _io + + def delete(self, storage_key): + if not self.exists(storage_key): + raise FileExistsError(f'The model {storage_key} not exist in the Cos.') + try: + rt = self.client.delete_bucket( + Bucket=self.Bucket, + Key=storage_key + ) + except Exception as e: + LOGGER.exception(e) + raise Exception(f"Delete model {storage_key} from Tencent COS failed: {e}") + + @staticmethod + def init_client(storage_address): + from qcloud_cos import CosS3Client, CosConfig + store_address = deepcopy(storage_address) + store_address.pop('Bucket') + return CosS3Client(CosConfig(**store_address)) + diff --git a/python/fate_flow/manager/model/handel/__init__.py b/python/fate_flow/manager/model/handel/__init__.py new file mode 100644 index 000000000..0b0b60763 --- /dev/null +++ b/python/fate_flow/manager/model/handel/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.manager.model.handel._base import IOHandle +from fate_flow.manager.model.handel._file import FileHandle +from fate_flow.manager.model.handel._mysql import MysqlHandel +from fate_flow.manager.model.handel._tencent_cos import TencentCosHandel + + +__all__ = ["IOHandle", "FileHandle", "MysqlHandel", "TencentCosHandel"] diff --git a/python/fate_flow/manager/model/handel/_base.py b/python/fate_flow/manager/model/handel/_base.py new file mode 100644 index 000000000..5bc6bbe16 --- /dev/null +++ b/python/fate_flow/manager/model/handel/_base.py @@ -0,0 +1,174 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import collections +import json +import os.path +import tarfile +from typing import Union, List + +from ruamel import yaml +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import Metadata +from fate_flow.errors.server_error import NoFoundModelOutput +from fate_flow.manager.model.model_meta import ModelMeta +from fate_flow.operation.job_saver import JobSaver + + +class IOHandle(object): + @property + def name(self): + return self._name + + @staticmethod + def storage_key(model_id, model_version, role, party_id, task_name, output_key): + return os.path.join(model_id, model_version, role, party_id, task_name, output_key) + + @staticmethod + def parse_storage_key(storage_key): + return storage_key.split(os.sep) + + def download(self, job_id=None, model_id=None, model_version=None, role=None, party_id=None, task_name=None, + output_key=None): + model_metas = ModelMeta.query(model_id=model_id, model_version=model_version, task_name=task_name, + output_key=output_key, role=role, party_id=party_id, job_id=job_id) + if not model_metas: + raise ValueError("No found model") + return self._download(storage_key=model_metas[0].f_storage_key) + + def upload(self, model_file: FileStorage, job_id, task_name, output_key, model_id, model_version, role, + party_id, type_name): + storage_key = self.storage_key(model_id, model_version, role, party_id, task_name, output_key) + metas = self._upload(model_file=model_file, storage_key=storage_key) + self.log_meta(metas, storage_key, job_id=job_id, task_name=task_name, model_id=model_id, type_name=type_name, + model_version=model_version, output_key=output_key, role=role, party_id=party_id) + + def save_as(self, storage_key, temp_path): + os.makedirs(os.path.dirname(temp_path), exist_ok=True) + return self._save_as(storage_key, temp_path) + + def load(self, file, storage_key, model_id, model_version, role, party_id, task_name, output_key): + metas = self._load(file=file, storage_key=storage_key) + self.log_meta(metas, storage_key, model_id=model_id, model_version=model_version, role=role, party_id=party_id, + task_name=task_name, output_key=output_key) + + def delete(self, **kwargs): + model_metas = ModelMeta.query(storage_engine=self.name, **kwargs) + if not model_metas: + raise NoFoundModelOutput(**kwargs) + for meta in model_metas: + try: + self._delete(storage_key=meta.f_storage_key) + except: + pass + return self.delete_meta(storage_engine=self.name, **kwargs) + + def log_meta(self, model_metas, storage_key, model_id, model_version, output_key, task_name, role, party_id, + job_id="", type_name=""): + model_info = { + "storage_key": storage_key, + "storage_engine": self.name, + "model_id": model_id, + "model_version": model_version, + "job_id": job_id, + "role": role, + "party_id": party_id, + "task_name": task_name, + "output_key": output_key, + "meta_data": model_metas, + "type_name": type_name + } + ModelMeta.save(**model_info) + + @staticmethod + def delete_meta(**kwargs): + return ModelMeta.delete(**kwargs) + + def meta_info(self, model_meta: Metadata): + execution_id = model_meta.model_overview.party.party_task_id + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + job = JobSaver.query_job(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id)[0] + _meta_info = { + "model_id": job.f_model_id, + "model_version": job.f_model_version, + "job_id": task.f_job_id, + "role": task.f_role, + "party_id": task.f_party_id, + "task_name": task.f_task_name, + "storage_engine": self.name + } + return _meta_info + + def read(self, job_id, role, party_id, task_name): + models = ModelMeta.query(job_id=job_id, role=role, party_id=party_id, task_name=task_name, reverse=True) + if not models: + raise NoFoundModelOutput(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + model_dict = {} + for model in models: + model_dict[model.f_output_key] = self._read(model.f_storage_key, model.f_meta_data) + return model_dict + + @property + def _name(self): + raise NotImplementedError() + + def _upload(self, **kwargs): + raise NotImplementedError() + + def _download(self, **kwargs): + raise NotImplementedError() + + def _read(self, storage_key, metas): + raise NotImplementedError() + + def _delete(self, storage_key): + raise NotImplementedError() + + def _save_as(self, storage_key, path): + raise NotImplementedError() + + def _load(self, file, storage_key): + raise NotImplementedError() + + @classmethod + def read_meta(cls, _tar: tarfile.TarFile) -> Union[Metadata, List[Metadata]]: + meta_list = [] + for name in _tar.getnames(): + if name.endswith("yaml"): + fp = _tar.extractfile(name).read() + meta = yaml.safe_load(fp) + meta_list.append(meta) + return meta_list + + @classmethod + def read_model(cls, _tar: tarfile.TarFile, metas): + model_cache = {} + for _meta in metas: + meta = Metadata(**_meta) + try: + fp = _tar.extractfile(meta.model_key).read() + _json_model = json.loads(fp) + if meta.index is None: + return _json_model + model_cache[meta.index] = _json_model + except Exception as e: + pass + return [model_cache[_k] for _k in sorted(model_cache)] + + @staticmethod + def update_meta(): + pass + + diff --git a/python/fate_flow/manager/model/handel/_file.py b/python/fate_flow/manager/model/handel/_file.py new file mode 100644 index 000000000..d60661e32 --- /dev/null +++ b/python/fate_flow/manager/model/handel/_file.py @@ -0,0 +1,74 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import os.path +import shutil +import tarfile + +from flask import send_file +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import FileStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.model.handel import IOHandle +from fate_flow.runtime.system_settings import MODEL_STORE_PATH + + +class FileHandle(IOHandle): + def __init__(self, engine_address: FileStorageSpec): + self.path = engine_address.path if engine_address.path else MODEL_STORE_PATH + + @property + def _name(self): + return ModelStorageEngine.FILE + + def _upload(self, model_file: FileStorage, storage_key): + _path = self._generate_model_storage_path(storage_key) + os.makedirs(os.path.dirname(_path), exist_ok=True) + model_file.save(_path) + model_metas = self.read_meta(self._tar_io(_path)) + return model_metas + + def _download(self, storage_key): + _p = self._generate_model_storage_path(storage_key) + return send_file(_p, download_name=os.path.basename(_p), as_attachment=True, mimetype='application/x-tar') + + def _save_as(self, storage_key, path): + _p = self._generate_model_storage_path(storage_key) + shutil.copy(_p, path) + return path + + def _load(self, file, storage_key): + _path = self._generate_model_storage_path(storage_key) + os.makedirs(os.path.dirname(_path), exist_ok=True) + shutil.copy(file, _path) + return self.read_meta(self._tar_io(_path)) + + def _read(self, storage_key, metas): + _p = self._generate_model_storage_path(storage_key) + _tar_io = self._tar_io(_p) + return self.read_model(_tar_io, metas) + + def _delete(self, storage_key): + _p = self._generate_model_storage_path(storage_key) + return os.remove(_p) + + @staticmethod + def _tar_io(path): + with open(path, "rb") as f: + return tarfile.open(fileobj=io.BytesIO(f.read())) + + def _generate_model_storage_path(self, storage_key): + return os.path.join(self.path, storage_key) diff --git a/python/fate_flow/manager/model/handel/_mysql.py b/python/fate_flow/manager/model/handel/_mysql.py new file mode 100644 index 000000000..4bc2865df --- /dev/null +++ b/python/fate_flow/manager/model/handel/_mysql.py @@ -0,0 +1,73 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import os +import tarfile + +from flask import send_file +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import MysqlStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.model.engine import MysqlModelStorage +from fate_flow.manager.model.handel import IOHandle + + +class MysqlHandel(IOHandle): + def __init__(self, engine_address: MysqlStorageSpec, decrypt_key=None): + self.engine = MysqlModelStorage(engine_address.dict(), decrypt_key=decrypt_key) + + @property + def _name(self): + return ModelStorageEngine.MYSQL + + def _upload(self, model_file: FileStorage, storage_key): + memory = io.BytesIO() + model_file.save(memory) + metas = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return metas + + def _download(self, storage_key): + memory = self.engine.read(storage_key) + memory.seek(0) + return send_file(memory, download_name=storage_key, as_attachment=True, mimetype='application/x-tar') + + def _read(self, storage_key, metas): + memory = self.engine.read(storage_key) + _tar_io = self._tar_io(memory) + return self.read_model(_tar_io, metas) + + def _delete(self, storage_key): + self.engine.delete(storage_key=storage_key) + + def _load(self, file, storage_key): + with open(file, "rb") as memory: + memory.seek(0) + model_meta = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return model_meta + + def _save_as(self, storage_key, path): + memory = self.engine.read(storage_key) + memory.seek(0) + with open(path, "wb") as f: + f.write(memory.read()) + return path + + @staticmethod + def _tar_io(memory): + memory.seek(0) + return tarfile.open(fileobj=memory) diff --git a/python/fate_flow/manager/model/handel/_tencent_cos.py b/python/fate_flow/manager/model/handel/_tencent_cos.py new file mode 100644 index 000000000..02d88fe2b --- /dev/null +++ b/python/fate_flow/manager/model/handel/_tencent_cos.py @@ -0,0 +1,71 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import tarfile + +from flask import send_file +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import TencentCosStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.model.engine import TencentCosStorage +from fate_flow.manager.model.handel import IOHandle + + +class TencentCosHandel(IOHandle): + def __init__(self, engine_address: TencentCosStorageSpec, decrypt_key: str = None): + self.engine = TencentCosStorage(engine_address.dict(), decrypt_key) + + @property + def _name(self): + return ModelStorageEngine.TENCENT_COS + + def _upload(self, model_file: FileStorage, storage_key): + memory = io.BytesIO() + model_file.save(memory) + metas = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return metas + + def _download(self, storage_key): + memory = self.engine.read(storage_key) + memory.seek(0) + return send_file(memory, as_attachment=True, download_name=storage_key, mimetype='application/x-tar') + + def _read(self, storage_key, metas): + memory = self.engine.read(storage_key) + _tar_io = self._tar_io(memory) + return self.read_model(_tar_io, metas) + + def _delete(self, storage_key): + return self.engine.delete(storage_key=storage_key) + + def _load(self, file, storage_key): + with open(file, "rb") as memory: + model_meta = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return model_meta + + def _save_as(self, storage_key, path): + memory = self.engine.read(storage_key) + memory.seek(0) + with open(path, "wb") as f: + f.write(memory.read()) + return path + + @staticmethod + def _tar_io(memory): + memory.seek(0) + return tarfile.open(fileobj=memory) diff --git a/python/fate_flow/manager/model/model_manager.py b/python/fate_flow/manager/model/model_manager.py new file mode 100644 index 000000000..8ac3016c9 --- /dev/null +++ b/python/fate_flow/manager/model/model_manager.py @@ -0,0 +1,93 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +from tempfile import TemporaryDirectory + +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import FileStorageSpec, MysqlStorageSpec, TencentCosStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.model.handel import FileHandle, MysqlHandel, TencentCosHandel +from fate_flow.manager.model.model_meta import ModelMeta +from fate_flow.runtime.system_settings import MODEL_STORE +from fate_flow.errors.server_error import NoFoundModelOutput + + +class PipelinedModel(object): + engine = MODEL_STORE.get("engine") + decrypt_key = MODEL_STORE.get("decrypt_key") + if engine == ModelStorageEngine.FILE: + handle = FileHandle(engine_address=FileStorageSpec(**MODEL_STORE.get(engine))) + elif engine == ModelStorageEngine.MYSQL: + handle = MysqlHandel(engine_address=MysqlStorageSpec(**MODEL_STORE.get(engine)), decrypt_key=decrypt_key) + elif engine == ModelStorageEngine.TENCENT_COS: + handle = TencentCosHandel(engine_address=TencentCosStorageSpec(**MODEL_STORE.get(engine)), + decrypt_key=decrypt_key) + else: + raise ValueError(f"Model storage engine {engine} is not supported.") + + @classmethod + def upload_model(cls, model_file: FileStorage, job_id: str, task_name, output_key, model_id, model_version, role, + party_id, type_name): + return cls.handle.upload(model_file, job_id, task_name, output_key, model_id, model_version, role, + party_id, type_name) + + @classmethod + def download_model(cls, **kwargs): + return cls.handle.download(**kwargs) + + @classmethod + def read_model(cls, job_id, role, party_id, task_name): + return cls.handle.read(job_id, role, party_id, task_name) + + @classmethod + def delete_model(cls, **kwargs): + return cls.handle.delete(**kwargs) + + @classmethod + def export_model(cls, model_id, model_version, role, party_id, dir_path): + _key_list = cls.get_model_storage_key(model_id=model_id, model_version=model_version, role=role, party_id=party_id) + if not _key_list: + raise NoFoundModelOutput(model_id=model_id, model_version=model_version, role=role, party_id=party_id) + with TemporaryDirectory() as temp_dir: + for _k in _key_list: + temp_path = os.path.join(temp_dir, _k) + cls.handle.save_as(storage_key=_k, temp_path=temp_path) + os.makedirs(dir_path, exist_ok=True) + shutil.make_archive(os.path.join(dir_path, f"{model_id}_{model_version}_{role}_{party_id}"), 'zip', temp_dir) + + @classmethod + def import_model(cls, model_id, model_version, path, temp_dir): + base_dir = os.path.dirname(path) + shutil.unpack_archive(path, base_dir, 'zip') + for dirpath, dirnames, filenames in os.walk(base_dir): + for filename in filenames: + model_path = os.path.join(dirpath, filename) + # exclude original model packs + if model_path != path: + _storage_key = model_path.lstrip(f"{temp_dir}{os.sep}") + _, _, role, party_id, task_name, output_key = cls.handle.parse_storage_key(_storage_key) + storage_key = cls.handle.storage_key(model_id, model_version, role, party_id, task_name, output_key) + cls.handle.load(model_path, storage_key, model_id, model_version, role=role, party_id=party_id, + task_name=task_name, output_key=output_key) + + @classmethod + def get_model_storage_key(cls, **kwargs): + _key_list = [] + _model_metas = ModelMeta.query(**kwargs) + for _meta in _model_metas: + _key_list.append(_meta.f_storage_key) + return _key_list diff --git a/python/fate_flow/manager/model/model_meta.py b/python/fate_flow/manager/model/model_meta.py new file mode 100644 index 000000000..696aca1e2 --- /dev/null +++ b/python/fate_flow/manager/model/model_meta.py @@ -0,0 +1,33 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.db.base_models import BaseModelOperate +from fate_flow.db.db_models import PipelineModelMeta +from fate_flow.utils.wraps_utils import filter_parameters + + +class ModelMeta(BaseModelOperate): + @classmethod + def save(cls, **meta_info): + cls._create_entity(PipelineModelMeta, meta_info) + + @classmethod + @filter_parameters() + def query(cls, **kwargs): + return cls._query(PipelineModelMeta, **kwargs) + + @classmethod + @filter_parameters() + def delete(cls, **kwargs): + return cls._delete(PipelineModelMeta, **kwargs) diff --git a/python/fate_flow/manager/model_manager.py b/python/fate_flow/manager/model_manager.py deleted file mode 100644 index 088ec5514..000000000 --- a/python/fate_flow/manager/model_manager.py +++ /dev/null @@ -1,163 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import os.path -import tarfile -import traceback - -from flask import send_file -from ruamel import yaml -from werkzeug.datastructures import FileStorage - -from fate_flow.db.base_models import BaseModelOperate -from fate_flow.db.db_models import PipelineModelMeta -from fate_flow.entity.model_spc import MLModelSpec -from fate_flow.settings import ( - CACHE_MODEL_STORE_PATH, - SOURCE_MODEL_STORE_PATH, - stat_logger, -) - - -class PipelinedModel(object): - def __init__(self, job_id, role, party_id, model_id: str = None, model_version: int = None, store_engine="file"): - self.job_id = job_id - self.model_id = model_id - self.model_version = model_version - self.role = role - self.party_id = party_id - self.handle = self._set_handle(store_engine) - self.meta_manager = ModelMeta(model_id, model_version, job_id, role, party_id) - - @classmethod - def _set_handle(cls, handle_type): - if handle_type == "file": - return FileHandle() - - def save_output_model(self, task_name, model_name, component, model_file: FileStorage): - self.handle.write(self.model_id, self.model_version, self.role, self.party_id, task_name, model_name, model_file) - self.meta_manager.save(task_name=task_name, component=component, model_name=model_name) - - def read_output_model(self, task_name, model_name): - return self.handle.read(self.model_id, self.model_version, self.role, self.party_id, task_name, model_name) - - def read_model_data(self, task_name): - model_data = {} - message = "success" - try: - model_metas = self.meta_manager.query(task_name=task_name) - for model_meta in model_metas: - model_data[model_meta.f_model_name] = self.handle.read_cache( - model_meta.f_model_id, model_meta.f_model_version, model_meta.f_role, model_meta.f_party_id, - model_meta.f_task_name, model_meta.f_model_name - ) - except Exception as e: - traceback.print_exc() - stat_logger.exception(e) - message = str(e) - return model_data, message - - -class ModelMeta(BaseModelOperate): - def __init__(self, model_id, model_version, job_id, role, party_id): - self.model_id = model_id - self.model_version = model_version - self.job_id = job_id - self.role = role - self.party_id = party_id - - def save(self, task_name, component, model_name): - meta_info = { - "job_id": self.job_id, - "model_id": self.model_id, - "model_version": self.model_version, - "role": self.role, - "party_id": self.party_id, - "task_name": task_name, - "component": component, - "model_name": model_name - } - self._create_entity(PipelineModelMeta, meta_info) - - def query(self, **kwargs): - return self._query(PipelineModelMeta, job_id=self.job_id, role=self.role, party_id=self.party_id, **kwargs) - - -class IOHandle: - def read(self, model_id, model_version, role, party_id, task_name, model_name): - ... - - def write(self, model_id, model_version, role, party_id, task_name, model_name, model_data): - ... - - -class FileHandle(IOHandle): - def __init__(self): - self.model_parser = FileModelParser() - - def write(self, model_id, model_version, role, party_id, task_name, model_name, model_file: FileStorage): - source_path = generate_model_storage_path(model_id, model_version, role, party_id, task_name, model_name) - os.makedirs(os.path.dirname(source_path), exist_ok=True) - model_file.save(source_path) - self.write_cache(model_id, model_version, role, party_id, task_name, model_name, source_path) - - def read(self, model_id, model_version, role, party_id, task_name, model_name): - model_path = os.path.join(SOURCE_MODEL_STORE_PATH, model_id, model_version, role, party_id, task_name, model_name) - return send_file(model_path, attachment_filename=model_name, as_attachment=True) - - def write_cache(self, model_id, model_version, role, party_id, task_name, model_name, source_path): - return self.model_parser.write_cache(model_id, model_version, role, party_id, task_name, model_name, source_path) - - def read_cache(self, model_id, model_version, role, party_id, task_name, model_name): - return self.model_parser.read_cache(model_id, model_version, role, party_id, task_name, model_name) - - -class FileModelParser: - @staticmethod - def write_cache(model_id, model_version, role, party_id, task_name, model_name, source_path): - path = generate_model_storage_path(model_id, model_version, role, party_id, task_name, model_name, cache=True) - os.makedirs(path, exist_ok=True) - tar = tarfile.open(source_path, "r:") - tar.extractall(path=path) - tar.close() - - @staticmethod - def read_cache(model_id, model_version, role, party_id, task_name, model_name): - base_path = generate_model_storage_path(model_id, model_version, role, party_id, task_name, model_name, cache=True) - model_meta = FileModelParser.get_model_meta(base_path) - model_cache = {} - for model in model_meta.party.models: - if model.file_format == "json": - model_file_name = os.path.join(base_path, model.name) - if os.path.exists(model_file_name): - with open(model_file_name, "r") as f: - model_cache[model.name] = json.load(f) - return model_cache - - @staticmethod - def get_model_meta(path) -> MLModelSpec: - for _file in os.listdir(path): - if _file.endswith("yaml"): - with open(os.path.join(path, _file), "r") as fp: - meta = yaml.safe_load(fp) - return MLModelSpec.parse_obj(meta) - - -def generate_model_storage_path(model_id, model_version, role, party_id, task_name, model_name=None, cache=False): - if not cache: - path = os.path.join(SOURCE_MODEL_STORE_PATH, model_id, str(model_version), role, party_id, task_name, model_name) - else: - path = os.path.join(CACHE_MODEL_STORE_PATH, model_id, str(model_version), role, party_id, task_name, model_name) - return path diff --git a/python/fate_flow/manager/pipeline/__init__.py b/python/fate_flow/manager/pipeline/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/manager/pipeline/pipeline.py b/python/fate_flow/manager/pipeline/pipeline.py new file mode 100644 index 000000000..16c8289f6 --- /dev/null +++ b/python/fate_flow/manager/pipeline/pipeline.py @@ -0,0 +1,83 @@ +from fate_flow.operation.job_saver import JobSaver +from fate_flow.utils.log_utils import schedule_logger + + + +def pipeline_dag_dependency(job): + component_list = [] + component_module, dependence_dict, component_need_run = {}, {}, {} + + tasks = job.f_dag["dag"].get("tasks") + for name, components in tasks.items(): + component_list.append(name) + component_module[name] = components["component_ref"] + dependence_dict[name] = [] + + for name, components in tasks.items(): + dependence_tasks = components["dependent_tasks"] + inputs = components.get("inputs", None) + if 'data' in inputs: + data_input = inputs["data"] + for data_key, data_dct in data_input.items(): + for _k, dataset in data_dct.items(): + if isinstance(dataset, list): + dataset = dataset[0] + up_component_name = dataset.get("producer_task") + # up_pos = component_list.index(up_component_name) + # up_component = components[up_pos] + # data_name = dataset.split(".", -1)[1] + # if up_component.get_output().get("data"): + # data_pos = up_component.get_output().get("data").index(data_name) + # else: + # data_pos = 0 + + if data_key == "data" or data_key == "train_data": + data_type = data_key + else: + data_type = "validate_data" + + dependence_dict[name].append({"component_name": up_component_name, + "type": data_type, + "up_output_info": [data_type, 0]}) + + input_keyword_type_mapping = {"model": "model", + "isometric_model": "model", + "cache": "cache"} + for keyword, v_type in input_keyword_type_mapping.items(): + if keyword in inputs: + input_list = inputs[keyword] + if not input_list or not isinstance(input_list, dict): + continue + # if isinstance(input_list, list): + # input_list = input_list[0] + for _k, _input in input_list.items(): + if isinstance(_input, list): + _input = _input[0] + up_component_name = _input.get("producer_task") + if up_component_name == "pipeline": + continue + # link_alias = _input.split(".", -1)[1] + # up_pos = component_list.index(up_component_name) + # up_component = self.components[up_pos] + # if up_component.get_output().get(v_type): + # dep_pos = up_component.get_output().get(v_type).index(link_alias) + # else: + dep_pos = 0 + dependence_dict[name].append({"component_name": up_component_name, + "type": v_type, + "up_output_info": [v_type, dep_pos]}) + + if not dependence_dict[name]: + del dependence_dict[name] + + tasks = JobSaver.query_task(job_id=job.f_job_id, party_id=job.f_party_id, role=job.f_role, only_latest=True) + for task in tasks: + need_run = task.f_component_parameters.get("ComponentParam", {}).get("need_run", True) + component_need_run[task.f_task_name] = need_run + + base_dependency = {"component_list": component_list, + "dependencies": dependence_dict, + "component_module": component_module, + "component_need_run": component_need_run } + + return base_dependency diff --git a/python/fate_flow/manager/service/__init__.py b/python/fate_flow/manager/service/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/manager/service/app_manager.py b/python/fate_flow/manager/service/app_manager.py new file mode 100644 index 000000000..8ecefac03 --- /dev/null +++ b/python/fate_flow/manager/service/app_manager.py @@ -0,0 +1,137 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import wraps + +from fate_flow.db.base_models import BaseModelOperate +from fate_flow.db.permission_models import AppInfo, PartnerAppInfo +from fate_flow.entity.types import AppType +from fate_flow.errors.server_error import NoFoundAppid, RoleTypeError +from fate_flow.runtime.system_settings import ADMIN_KEY, CLIENT_AUTHENTICATION, APP_TOKEN_LENGTH, SITE_AUTHENTICATION, \ + PARTY_ID +from fate_flow.utils.base_utils import generate_random_id +from fate_flow.utils.wraps_utils import filter_parameters, switch_function, check_permission + + +class AppManager(BaseModelOperate): + @classmethod + def init(cls): + if CLIENT_AUTHENTICATION or SITE_AUTHENTICATION: + if cls.query_app(app_name="admin", init=True): + cls._delete(AppInfo, app_name="admin") + cls.create_app(app_name="admin", app_id="admin", app_token=ADMIN_KEY, app_type="admin", init=True) + app_info = cls.create_app(app_name=PARTY_ID, app_id=PARTY_ID, app_type=AppType.SITE, init=True) + if app_info: + cls.create_partner_app(party_id=PARTY_ID, app_id=app_info.get("app_id"), + app_token=app_info.get("app_token")) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @check_permission(operate="create", types="client") + def create_app(cls, app_type, app_name, app_id=None, app_token=None, init=True): + if not app_id: + app_id = cls.generate_app_id() + if not app_token: + app_token = cls.generate_app_token() + app_info = { + "app_name": app_name, + "app_id": app_id, + "app_token": app_token, + "app_type": app_type + } + status = cls._create_entity(AppInfo, app_info) + if status: + return app_info + else: + return {} + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def create_partner_app(cls, party_id, app_id=None, app_token=None): + app_info = { + "party_id": party_id, + "app_id": app_id, + "app_token": app_token, + } + status = cls._create_entity(PartnerAppInfo, app_info) + if status: + return app_info + else: + return {} + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + @check_permission(operate="delete", types="client") + def delete_app(cls, init=False, **kwargs): + return cls._delete(AppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + def delete_partner_app(cls, init=False, **kwargs): + return cls._delete(PartnerAppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + @check_permission(operate="query", types="client") + def query_app(cls, init=False, **kwargs): + return cls._query(AppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + def query_partner_app(cls, **kwargs): + return cls._query(PartnerAppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def generate_app_id(cls, length=8): + app_id = generate_random_id(length=length, only_number=True) + if cls.query_app(app_id=app_id): + cls.generate_app_id() + else: + return app_id + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def generate_app_token(cls, length=APP_TOKEN_LENGTH): + return generate_random_id(length=length) + + @staticmethod + def check_app_id(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if kwargs.get("app_id"): + if not AppManager.query_app(app_id=kwargs.get("app_id")): + raise NoFoundAppid(app_id=kwargs.get("app_id")) + return func(*args, **kwargs) + return _wrapper + + @staticmethod + def check_app_type(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if kwargs.get("app_id"): + app_info = AppManager.query_app(app_id=kwargs.get("app_id")) + if not app_info: + raise NoFoundAppid(app_id=kwargs.get("app_id")) + role = kwargs.get("role") + if role == "super_client": + role = "client" + if role != app_info[0].f_app_type: + raise RoleTypeError(role=kwargs.get("role")) + return func(*args, **kwargs) + return _wrapper \ No newline at end of file diff --git a/python/fate_flow/manager/service/output_manager.py b/python/fate_flow/manager/service/output_manager.py new file mode 100644 index 000000000..e26b8824d --- /dev/null +++ b/python/fate_flow/manager/service/output_manager.py @@ -0,0 +1,30 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +from fate_flow.db.base_models import BaseModelOperate +from fate_flow.db.db_models import TrackingOutputInfo +from fate_flow.utils.wraps_utils import filter_parameters + + +class OutputDataTracking(BaseModelOperate): + @classmethod + def create(cls, entity_info): + cls._create_entity(TrackingOutputInfo, entity_info) + + @classmethod + @filter_parameters() + def query(cls, reverse=False, **kwargs) -> List[TrackingOutputInfo]: + return cls._query(TrackingOutputInfo, reverse=reverse, order_by="index", **kwargs) diff --git a/python/fate_flow/manager/service/provider_manager.py b/python/fate_flow/manager/service/provider_manager.py new file mode 100644 index 000000000..7f7ff9e5b --- /dev/null +++ b/python/fate_flow/manager/service/provider_manager.py @@ -0,0 +1,175 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import sys +from typing import Union + +from fate_flow.db import ProviderInfo, ComponentInfo +from fate_flow.db.base_models import DB, BaseModelOperate +from fate_flow.entity.spec.flow import ProviderSpec, LocalProviderSpec, DockerProviderSpec, K8sProviderSpec +from fate_flow.entity.types import ProviderDevice +from fate_flow.hub.flow_hub import FlowHub +from fate_flow.hub.provider import EntrypointABC +from fate_flow.runtime.system_settings import DEFAULT_FATE_PROVIDER_PATH, DEFAULT_PROVIDER, FATE_FLOW_PROVIDER_PATH +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.utils.log import getLogger +from fate_flow.utils.version import get_versions, get_default_fate_version, get_flow_version +from fate_flow.utils.wraps_utils import filter_parameters + +stat_logger = getLogger("fate_flow_stat") + +class ProviderManager(BaseModelOperate): + @classmethod + def get_provider_by_provider_name(cls, provider_name) -> ComponentProvider: + name, version, device = cls.parser_provider_name(provider_name) + provider_list = [provider_info for provider_info in cls.query_provider(name=name, version=version, device=device)] + if not provider_list: + raise ValueError(f"Query provider info failed: {provider_name}") + provider_info = provider_list[0] + return cls.get_provider(provider_info.f_name, provider_info.f_device, provider_info.f_version, + provider_info.f_metadata) + + @classmethod + def get_provider(cls, name, device, version, metadata, check=False) -> Union[ComponentProvider, None]: + if device == ProviderDevice.LOCAL: + metadata = LocalProviderSpec(check, **metadata) + elif device == ProviderDevice.DOCKER: + metadata = DockerProviderSpec(check, **metadata) + elif device == ProviderDevice.K8S: + metadata = K8sProviderSpec(check, **metadata) + else: + return None + return ComponentProvider(ProviderSpec(name=name, device=device, version=version, metadata=metadata)) + + @classmethod + @DB.connection_context() + def register_provider(cls, provider: ComponentProvider): + provider_info = ProviderInfo() + provider_info.f_provider_name = provider.provider_name + provider_info.f_name = provider.name + provider_info.f_device = provider.device + provider_info.f_version = provider.version + provider_info.f_metadata = provider.metadata.dict() + operator_type = cls.safe_save(ProviderInfo, defaults=provider_info.to_dict(), + f_provider_name=provider.provider_name) + # todo: load entrypoint、components、params... + # load components + cls.register_component(provider) + return operator_type + + @classmethod + def register_component(cls, provider: ComponentProvider): + entrypoint = cls.load_entrypoint(provider) + component_list = [] + if entrypoint: + component_list = entrypoint.component_list + + for component_name in component_list: + component = ComponentInfo() + component.f_provider_name = provider.provider_name + component.f_name = provider.name + component.f_device = provider.device + component.f_version = provider.version + component.f_component_name = component_name + cls.safe_save(ComponentInfo, defaults=component.to_dict(), **component.to_dict()) + + @classmethod + @filter_parameters() + def query_provider(cls, **kwargs): + return cls._query(ProviderInfo, **kwargs) + + @classmethod + @filter_parameters() + def delete_provider(cls, **kwargs): + result = cls._delete(ProviderInfo, **kwargs) + cls.delete_provider_component_info(**kwargs) + return result + + @classmethod + @filter_parameters() + def delete_provider_component_info(cls, **kwargs): + result = cls._delete(ComponentInfo, **kwargs) + return result + + @classmethod + def register_default_providers(cls): + # register fate flow + cls.register_provider(cls.get_fate_flow_provider()) + # try to register fate + try: + cls.register_provider(cls.get_default_fate_provider()) + except Exception as e: + stat_logger.exception(e) + + @classmethod + def get_all_components(cls): + component_list = cls._query(ComponentInfo, force=True) + return list(set([component.f_component_name for component in component_list])) + + @classmethod + def get_fate_flow_provider(cls): + return cls.get_provider( + name="fate_flow", + version=get_flow_version(), + device=ProviderDevice.LOCAL, + metadata={ + "path": FATE_FLOW_PROVIDER_PATH, + "venv": sys.executable + }) + + @classmethod + def get_default_fate_provider(cls): + return cls.get_provider( + name="fate", + version=get_default_fate_version(), + device=ProviderDevice.LOCAL, + metadata={ + "path": DEFAULT_FATE_PROVIDER_PATH, + "venv": sys.executable + }) + + @classmethod + def generate_provider_name(cls, name, version, device): + return f"{name}:{version}@{device}" + + @classmethod + def parser_provider_name(cls, provider_name): + if not provider_name: + return None, None, None + try: + return provider_name.split(":")[0], provider_name.split(":")[1].split("@")[0], provider_name.split(":")[1].split("@")[1] + except: + raise ValueError(f"Provider format should be name:version@device, please check: {provider_name}") + + @classmethod + def check_provider_name(cls, provider_name): + name, version, device = cls.parser_provider_name(provider_name) + if not name or name == "*": + name = DEFAULT_PROVIDER.get("name") + if not version or version == "*": + if not DEFAULT_PROVIDER.get("version"): + DEFAULT_PROVIDER["version"] = get_versions().get(name.upper()) + version = DEFAULT_PROVIDER.get("version") + if not device or device == "*": + device = DEFAULT_PROVIDER.get("device") + provider_info = cls.query_provider(name=name, version=version, device=device) + if not provider_info: + raise ValueError(f"Not found provider[{cls.generate_provider_name(name, version, device)}]") + return cls.generate_provider_name(name, version, device) + + @staticmethod + def load_entrypoint(provider) -> Union[None, EntrypointABC]: + return FlowHub.load_provider_entrypoint(provider) diff --git a/python/fate_flow/manager/resource_manager.py b/python/fate_flow/manager/service/resource_manager.py similarity index 72% rename from python/fate_flow/manager/resource_manager.py rename to python/fate_flow/manager/service/resource_manager.py index 0243201a0..89d0608c8 100644 --- a/python/fate_flow/manager/resource_manager.py +++ b/python/fate_flow/manager/service/resource_manager.py @@ -16,14 +16,16 @@ from pydantic import typing from fate_flow.db.base_models import DB -from fate_flow.db.db_models import EngineRegistry, Job -from fate_flow.entity.engine_types import EngineType -from fate_flow.entity.types import ResourceOperation +from fate_flow.db.db_models import EngineRegistry, Job, Task +from fate_flow.entity.types import EngineType, ResourceOperation from fate_flow.runtime.job_default_config import JobDefaultConfig -from fate_flow.settings import stat_logger, IGNORE_RESOURCE_ROLES, ENGINES +from fate_flow.runtime.system_settings import IGNORE_RESOURCE_ROLES, ENGINES from fate_flow.utils import engine_utils, base_utils, job_utils +from fate_flow.utils.log import getLogger from fate_flow.utils.log_utils import schedule_logger +stat_logger = getLogger() + class ResourceManager(object): @classmethod @@ -36,9 +38,8 @@ def initialize(cls): @classmethod @DB.connection_context() def register_engine(cls, engine_type, engine_name, engine_config): - nodes = engine_config.get("nodes", 1) - cores = engine_config.get("cores_per_node", 0) * nodes * JobDefaultConfig.total_cores_overweight_percent - memory = engine_config.get("memory_per_node", 0) * nodes * JobDefaultConfig.total_memory_overweight_percent + cores = engine_config.get("cores", 0) + memory = engine_config.get("memory", 0) filters = [EngineRegistry.f_engine_type == engine_type, EngineRegistry.f_engine_name == engine_name] resources = EngineRegistry.select().where(*filters) if resources: @@ -51,7 +52,6 @@ def register_engine(cls, engine_type, engine_name, engine_config): cores - resource.f_cores) update_fields[EngineRegistry.f_remaining_memory] = EngineRegistry.f_remaining_memory + ( memory - resource.f_memory) - update_fields[EngineRegistry.f_nodes] = nodes operate = EngineRegistry.update(update_fields).where(*filters) update_status = operate.execute() > 0 if update_status: @@ -69,14 +69,12 @@ def register_engine(cls, engine_type, engine_name, engine_config): resource.f_memory = memory resource.f_remaining_cores = cores resource.f_remaining_memory = memory - resource.f_nodes = nodes try: resource.save(force_insert=True) except Exception as e: stat_logger.warning(e) stat_logger.info(f"create {engine_type} engine {engine_name} registration information") - @classmethod def apply_for_job_resource(cls, job_id, role, party_id): return cls.resource_for_job(job_id=job_id, role=role, party_id=party_id, operation_type=ResourceOperation.APPLY) @@ -90,12 +88,11 @@ def return_job_resource(cls, job_id, role, party_id): @DB.connection_context() def resource_for_job(cls, job_id, role, party_id, operation_type: ResourceOperation): operate_status = False - cores, memory = cls.calculate_job_resource(job_id=job_id, role=role, party_id=party_id) + cores, memory = cls.query_job_resource(job_id=job_id, role=role, party_id=party_id) engine_name = ENGINES.get(EngineType.COMPUTING) try: with DB.atomic(): updates = { - Job.f_engine_type: EngineType.COMPUTING, Job.f_engine_name: engine_name, Job.f_cores: cores, Job.f_memory: memory, @@ -106,8 +103,6 @@ def resource_for_job(cls, job_id, role, party_id, operation_type: ResourceOperat Job.f_party_id == party_id, ] if operation_type is ResourceOperation.APPLY: - updates[Job.f_remaining_cores] = cores - updates[Job.f_remaining_memory] = memory updates[Job.f_resource_in_use] = True updates[Job.f_apply_resource_time] = base_utils.current_timestamp() filters.append(Job.f_resource_in_use == False) @@ -126,7 +121,6 @@ def resource_for_job(cls, job_id, role, party_id, operation_type: ResourceOperat memory=memory, operation_type=operation_type, ) - filters.append(EngineRegistry.f_engine_type == EngineType.COMPUTING) filters.append(EngineRegistry.f_engine_name == engine_name) operate = EngineRegistry.update(updates).where(*filters) apply_status = operate.execute() > 0 @@ -162,20 +156,48 @@ def return_task_resource(cls, **task_info): @classmethod @DB.connection_context() def resource_for_task(cls, task_info, operation_type): - cores_per_task, memory_per_task = cls.calculate_task_resource(task_info=task_info) - schedule_logger(task_info["job_id"]).info(f"cores_per_task:{cores_per_task}, memory_per_task:{memory_per_task}") + cores_per_task, memory_per_task = cls.query_task_resource(task_info=task_info) + schedule_logger(task_info["job_id"]).info(f"{operation_type} cores_per_task:{cores_per_task}, memory_per_task:{memory_per_task}") + operate_status = False if cores_per_task or memory_per_task: - filters, updates = cls.update_resource_sql(resource_model=Job, - cores=cores_per_task, - memory=memory_per_task, - operation_type=operation_type, - ) - filters.append(Job.f_job_id == task_info["job_id"]) - filters.append(Job.f_role == task_info["role"]) - filters.append(Job.f_party_id == task_info["party_id"]) - filters.append(Job.f_resource_in_use == True) - operate = Job.update(updates).where(*filters) - operate_status = operate.execute() > 0 + try: + with DB.atomic(): + updates = {} + filters = [ + Task.f_job_id == task_info["job_id"], + Task.f_role == task_info["role"], + Task.f_party_id == task_info["party_id"], + Task.f_task_id == task_info["task_id"], + Task.f_task_version == task_info["task_version"] + ] + if operation_type is ResourceOperation.APPLY: + updates[Task.f_resource_in_use] = True + filters.append(Task.f_resource_in_use == False) + elif operation_type is ResourceOperation.RETURN: + updates[Task.f_resource_in_use] = False + filters.append(Task.f_resource_in_use == True) + operate = Task.update(updates).where(*filters) + record_status = operate.execute() > 0 + if not record_status: + raise RuntimeError(f"record task {task_info['task_id']} {task_info['task_version']} resource" + f"{operation_type} failed on {task_info['role']} {task_info['party_id']}") + filters, updates = cls.update_resource_sql(resource_model=Job, + cores=cores_per_task, + memory=memory_per_task, + operation_type=operation_type, + ) + filters.append(Job.f_job_id == task_info["job_id"]) + filters.append(Job.f_role == task_info["role"]) + filters.append(Job.f_party_id == task_info["party_id"]) + # filters.append(Job.f_resource_in_use == True) + operate = Job.update(updates).where(*filters) + operate_status = operate.execute() > 0 + if not operate_status: + raise RuntimeError(f"record task {task_info['task_id']} {task_info['task_version']} job resource " + f"{operation_type} failed on {task_info['role']} {task_info['party_id']}") + except Exception as e: + schedule_logger(task_info["job_id"]).warning(e) + operate_status = False else: operate_status = True if operate_status: @@ -194,30 +216,20 @@ def resource_for_task(cls, task_info, operation_type): return operate_status @classmethod - def calculate_job_resource(cls, job_id, role, party_id): - cores = 0 - memory = 0 - if role in IGNORE_RESOURCE_ROLES: - return cores, memory - task_cores, task_parallelism = job_utils.get_job_resource_info(job_id, role, party_id) - if not task_cores: - task_cores = JobDefaultConfig.task_cores - if not task_parallelism: - task_parallelism = JobDefaultConfig.task_parallelism - - cores = int(task_cores) * int(task_parallelism) - return cores, memory + def query_job_resource(cls, job_id, role, party_id): + cores, memory = job_utils.get_job_resource_info(job_id, role, party_id) + return int(cores), memory @classmethod - def calculate_task_resource(cls, task_info: dict = None): + def query_task_resource(cls, task_info: dict = None): cores_per_task = 0 memory_per_task = 0 if task_info["role"] in IGNORE_RESOURCE_ROLES: return cores_per_task, memory_per_task - cores_per_task, task_parallelism = job_utils.get_job_resource_info(task_info["job_id"], task_info["role"], task_info["party_id"]) - if not cores_per_task: - cores_per_task = JobDefaultConfig.task_cores - return cores_per_task, memory_per_task + task_cores, memory = job_utils.get_task_resource_info( + task_info["job_id"], task_info["role"], task_info["party_id"], task_info["task_id"], task_info["task_version"] + ) + return task_cores, memory @classmethod def update_resource_sql(cls, resource_model: typing.Union[EngineRegistry, Job], cores, memory, operation_type: ResourceOperation): @@ -229,7 +241,9 @@ def update_resource_sql(cls, resource_model: typing.Union[EngineRegistry, Job], updates = {resource_model.f_remaining_cores: resource_model.f_remaining_cores - cores, resource_model.f_remaining_memory: resource_model.f_remaining_memory - memory} elif operation_type is ResourceOperation.RETURN: - filters = [] + filters = [ + resource_model.f_remaining_cores + cores <= resource_model.f_cores + ] updates = {resource_model.f_remaining_cores: resource_model.f_remaining_cores + cores, resource_model.f_remaining_memory: resource_model.f_remaining_memory + memory} else: diff --git a/python/fate_flow/manager/service/service_manager.py b/python/fate_flow/manager/service/service_manager.py new file mode 100644 index 000000000..65bcf7080 --- /dev/null +++ b/python/fate_flow/manager/service/service_manager.py @@ -0,0 +1,517 @@ +# +# Copyright 2021 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +import atexit +import json +import socket +import time +from functools import wraps +from pathlib import Path +from queue import Queue +from threading import Thread +from urllib import parse + +from kazoo.client import KazooClient +from kazoo.exceptions import NodeExistsError, NoNodeError, ZookeeperError +from kazoo.security import make_digest_acl +from shortuuid import ShortUUID + +from fate_flow.db import ServiceRegistryInfo, ServerRegistryInfo +from fate_flow.db.base_models import DB +from fate_flow.entity.types import FlowInstance +from fate_flow.errors.zookeeper_error import ServiceNotSupported, ServicesError, ZooKeeperNotConfigured, \ + MissingZooKeeperUsernameOrPassword, ZooKeeperBackendError +from fate_flow.runtime.reload_config_base import ReloadConfigBase +from fate_flow.runtime.system_settings import RANDOM_INSTANCE_ID, HOST, HTTP_PORT, GRPC_PORT, ZOOKEEPER_REGISTRY, \ + ZOOKEEPER, USE_REGISTRY, NGINX_HOST, NGINX_HTTP_PORT, FATE_FLOW_MODEL_TRANSFER_ENDPOINT, SERVICE_CONF_NAME +from fate_flow.utils import conf_utils, file_utils +from fate_flow.utils.log import getLogger +from fate_flow.utils.version import get_flow_version + +stat_logger = getLogger("fate_flow_stat") + +model_download_endpoint = f'http://{NGINX_HOST}:{NGINX_HTTP_PORT}{FATE_FLOW_MODEL_TRANSFER_ENDPOINT}' + +instance_id = ShortUUID().random(length=8) if RANDOM_INSTANCE_ID else f'flow-{HOST}-{HTTP_PORT}' +server_instance = ( + f'{HOST}:{GRPC_PORT}', + json.dumps({ + 'instance_id': instance_id, + 'timestamp': round(time.time() * 1000), + 'version': get_flow_version() or '', + 'host': HOST, + 'grpc_port': GRPC_PORT, + 'http_port': HTTP_PORT, + }), +) + + +def check_service_supported(method): + """Decorator to check if `service_name` is supported. + The attribute `supported_services` MUST be defined in class. + The first and second arguments of `method` MUST be `self` and `service_name`. + + :param Callable method: The class method. + :return: The inner wrapper function. + :rtype: Callable + """ + @wraps(method) + def magic(self, service_name, *args, **kwargs): + if service_name not in self.supported_services: + raise ServiceNotSupported(service_name=service_name) + return method(self, service_name, *args, **kwargs) + return magic + + +class ServicesDB(abc.ABC): + """Database for storage service urls. + Abstract base class for the real backends. + + """ + @property + @abc.abstractmethod + def supported_services(self): + """The names of supported services. + The returned list SHOULD contain `fateflow` (model download) and `servings` (FATE-Serving). + + :return: The service names. + :rtype: list + """ + pass + + @abc.abstractmethod + def _insert(self, service_name, service_url, value=''): + pass + + @check_service_supported + def insert(self, service_name, service_url, value=''): + """Insert a service url to database. + + :param str service_name: The service name. + :param str service_url: The service url. + :return: None + """ + try: + self._insert(service_name, service_url, value) + except ServicesError as e: + stat_logger.exception(e) + + @abc.abstractmethod + def _delete(self, service_name, service_url): + pass + + @check_service_supported + def delete(self, service_name, service_url): + """Delete a service url from database. + + :param str service_name: The service name. + :param str service_url: The service url. + :return: None + """ + try: + self._delete(service_name, service_url) + except ServicesError as e: + stat_logger.exception(e) + + def register_model(self, party_model_id, model_version): + # todo + pass + + def unregister_model(self, party_model_id, model_version): + """Call `self.delete` for delete a service url from database. + Currently, only `fateflow` (model download) urls are supported. + + :param str party_model_id: The party model id, `#` will be replaced with `_`. + :param str model_version: The model version. + :return: None + """ + # todo + pass + + def register_flow(self): + """Call `self.insert` for insert the flow server address to databae. + + :return: None + """ + self.insert('flow-server', *server_instance) + + def unregister_flow(self): + """Call `self.delete` for delete the flow server address from databae. + + :return: None + """ + self.delete('flow-server', server_instance[0]) + + @abc.abstractmethod + def _get_urls(self, service_name, with_values=False): + pass + + @check_service_supported + def get_urls(self, service_name, with_values=False): + """Query service urls from database. The urls may belong to other nodes. + Currently, only `fateflow` (model download) urls and `servings` (FATE-Serving) urls are supported. + `fateflow` is a url containing scheme, host, port and path, + while `servings` only contains host and port. + + :param str service_name: The service name. + :return: The service urls. + :rtype: list + """ + try: + return self._get_urls(service_name, with_values) + except ServicesError as e: + stat_logger.exception(e) + return [] + + def register_models(self): + """Register all service urls of each model to database on this node. + + :return: None + """ + # todo: + pass + + def unregister_models(self): + """Unregister all service urls of each model to database on this node. + + :return: None + """ + # todo + pass + + def get_servers(self, to_dict=False): + servers = {} + for znode, value in self.get_urls('flow-server', True): + instance = FlowInstance(**json.loads(value)) + _id = instance.instance_id + if to_dict: + instance = instance.to_dict() + servers[_id] = instance + return servers + + +class ZooKeeperDB(ServicesDB): + """ZooKeeper Database + + """ + znodes = ZOOKEEPER_REGISTRY + supported_services = znodes.keys() + + def __init__(self): + hosts = ZOOKEEPER.get('hosts') + if not isinstance(hosts, list) or not hosts: + raise ZooKeeperNotConfigured() + + client_kwargs = {'hosts': hosts} + + use_acl = ZOOKEEPER.get('use_acl', False) + if use_acl: + username = ZOOKEEPER.get('user') + password = ZOOKEEPER.get('password') + if not username or not password: + raise MissingZooKeeperUsernameOrPassword() + + client_kwargs['default_acl'] = [make_digest_acl(username, password, all=True)] + client_kwargs['auth_data'] = [('digest', ':'.join([username, password]))] + + try: + # `KazooClient` is thread-safe, it contains `_thread.RLock` and can not be pickle. + # So be careful when using `self.client` outside the class. + self.client = KazooClient(**client_kwargs) + self.client.start() + except ZookeeperError as e: + raise ZooKeeperBackendError(error_message=repr(e)) + + atexit.register(self.client.stop) + + self.znodes_list = Queue() + Thread(target=self._watcher).start() + + def _insert(self, service_name, service_url, value=''): + znode = self._get_znode_path(service_name, service_url) + value = value.encode('utf-8') + + try: + self.client.create(znode, value, ephemeral=True, makepath=True) + except NodeExistsError: + stat_logger.warning(f'Znode `{znode}` exists, add it to watch list.') + self.znodes_list.put((znode, value)) + except ZookeeperError as e: + raise ZooKeeperBackendError(error_message=repr(e)) + + def _delete(self, service_name, service_url): + znode = self._get_znode_path(service_name, service_url) + + try: + self.client.delete(znode) + except NoNodeError: + stat_logger.warning(f'Znode `{znode}` not found, ignore deletion.') + except ZookeeperError as e: + raise ZooKeeperBackendError(error_message=repr(e)) + + def _get_znode_path(self, service_name, service_url): + """Get the znode path by service_name. + + :param str service_name: The service name. + :param str service_url: The service url. + :return: The znode path composed of `self.znodes[service_name]` and escaped `service_url`. + :rtype: str + + """ + + return '/'.join([self.znodes[service_name], parse.quote(service_url, safe='')]) + + def _get_urls(self, service_name, with_values=False): + try: + _urls = self.client.get_children(self.znodes[service_name]) + except ZookeeperError as e: + raise ZooKeeperBackendError(error_message=repr(e)) + + urls = [] + + for url in _urls: + url = parse.unquote(url) + data = '' + znode = self._get_znode_path(service_name, url) + + if service_name == 'servings': + url = parse.urlparse(url).netloc or url + + if with_values: + try: + data = self.client.get(znode) + except NoNodeError: + stat_logger.warning(f'Znode `{znode}` not found, return empty value.') + except ZookeeperError as e: + raise ZooKeeperBackendError(error_message=repr(e)) + else: + data = data[0].decode('utf-8') + + urls.append((url, data) if with_values else url) + + return urls + + def _watcher(self): + while True: + znode, value = self.znodes_list.get() + + try: + self.client.create(znode, value, ephemeral=True, makepath=True) + except NodeExistsError: + stat = self.client.exists(znode) + + if stat is not None: + if stat.owner_session_id is None: + stat_logger.warning(f'Znode `{znode}` is not an ephemeral node.') + continue + if stat.owner_session_id == self.client.client_id[0]: + stat_logger.warning(f'Duplicate znode `{znode}`.') + continue + + self.znodes_list.put((znode, value)) + + +class FallbackDB(ServicesDB): + """Fallback Database. + This class get the service url from `conf/service_conf.yaml` + It cannot insert or delete the service url. + + """ + supported_services = ( + 'fateflow', + 'flow-server', + 'servings', + ) + + def _insert(self, *args, **kwargs): + pass + + def _delete(self, *args, **kwargs): + pass + + def _get_urls(self, service_name, with_values=False): + if service_name == 'fateflow': + return [(model_download_endpoint, '')] if with_values else [model_download_endpoint] + if service_name == 'flow-server': + return [server_instance] if with_values else [server_instance[0]] + + urls = getattr(ServerRegistry, service_name.upper(), []) + if isinstance(urls, dict): + urls = urls.get('hosts', []) + if not isinstance(urls, list): + urls = [urls] + return [(url, '') for url in urls] if with_values else urls + + +class ServerRegistry(ReloadConfigBase): + @classmethod + def load(cls): + cls.load_server_info_from_conf() + cls.load_server_info_from_db() + + @classmethod + def register(cls, server_name, host, port, protocol): + server_name = server_name.upper() + server_info = { + "host": host, + "port": port, + "protocol": protocol + } + cls.save_server_info_to_db(server_name, host, port=port, protocol=protocol) + setattr(cls, server_name, server_info) + server_info.update({"server_name": server_name}) + return server_info + + @classmethod + def delete_server_from_db(cls, server_name): + operate = ServerRegistryInfo.delete().where(ServerRegistryInfo.f_server_name == server_name.upper()) + return operate.execute() + + @classmethod + def parameter_check(cls, service_info): + if "host" in service_info and "port" in service_info: + cls.connection_test(service_info.get("host"), service_info.get("port")) + + @classmethod + def connection_test(cls, ip, port): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = s.connect_ex((ip, port)) + if result != 0: + raise ConnectionRefusedError(f"connection refused: host {ip}, port {port}") + + @classmethod + def query(cls, service_name, default=None): + service_info = getattr(cls, service_name, default) + if not service_info: + service_info = conf_utils.get_base_config(service_name, default) + return service_info + + @classmethod + @DB.connection_context() + def query_server_info_from_db(cls, server_name=None) -> [ServerRegistryInfo]: + if server_name: + server_list = ServerRegistryInfo.select().where(ServerRegistryInfo.f_server_name == server_name.upper()) + else: + server_list = ServerRegistryInfo.select() + return [server for server in server_list] + + @classmethod + @DB.connection_context() + def load_server_info_from_db(cls): + for server in cls.query_server_info_from_db(): + server_info = { + "host": server.f_host, + "port": server.f_port, + "protocol": server.f_protocol + } + setattr(cls, server.f_server_name.upper(), server_info) + + @classmethod + def load_server_info_from_conf(cls): + path = Path(file_utils.get_fate_flow_directory()) / 'conf' / SERVICE_CONF_NAME + conf = file_utils.load_yaml_conf(path) + if not isinstance(conf, dict): + raise ValueError('invalid config file') + + local_path = path.with_name(f'local.{SERVICE_CONF_NAME}') + if local_path.exists(): + local_conf = file_utils.load_yaml_conf(local_path) + if not isinstance(local_conf, dict): + raise ValueError('invalid local config file') + conf.update(local_conf) + for k, v in conf.items(): + if isinstance(v, dict): + setattr(cls, k.upper(), v) + + @classmethod + @DB.connection_context() + def save_server_info_to_db(cls, server_name, host, port, protocol="http"): + server_info = { + "f_server_name": server_name, + "f_host": host, + "f_port": port, + "f_protocol": protocol + } + entity_model, status = ServerRegistryInfo.get_or_create( + f_server_name=server_name, + defaults=server_info) + if status is False: + for key in server_info: + setattr(entity_model, key, server_info[key]) + entity_model.save(force_insert=False) + + +class ServiceRegistry: + @classmethod + @DB.connection_context() + def load_service(cls, server_name, service_name) -> [ServiceRegistryInfo]: + server_name = server_name.upper() + service_registry_list = ServiceRegistryInfo.query(server_name=server_name, service_name=service_name) + return [service for service in service_registry_list] + + @classmethod + @DB.connection_context() + def save_service_info(cls, server_name, service_name, uri, method="POST", server_info=None, params=None, data=None, headers=None, protocol="http"): + server_name = server_name.upper() + if not server_info: + server_list = ServerRegistry.query_server_info_from_db(server_name=server_name) + if not server_list: + raise Exception(f"no found server {server_name}") + server_info = server_list[0] + url = f"{server_info.f_protocol}://{server_info.f_host}:{server_info.f_port}{uri}" + else: + url = f"{server_info.get('protocol', protocol)}://{server_info.get('host')}:{server_info.get('port')}{uri}" + service_info = { + "f_server_name": server_name, + "f_service_name": service_name, + "f_url": url, + "f_method": method, + "f_params": params if params else {}, + "f_data": data if data else {}, + "f_headers": headers if headers else {} + } + entity_model, status = ServiceRegistryInfo.get_or_create( + f_server_name=server_name, + f_service_name=service_name, + defaults=service_info) + if status is False: + for key in service_info: + setattr(entity_model, key, service_info[key]) + entity_model.save(force_insert=False) + + @classmethod + @DB.connection_context() + def delete(cls, server_name, service_name): + server_name = server_name.upper() + operate = ServiceRegistryInfo.delete().where(ServiceRegistryInfo.f_server_name == server_name.upper(), + ServiceRegistryInfo.f_service_name == service_name) + return operate.execute() + + +def service_db(): + """Initialize services database. + Currently only ZooKeeper is supported. + + :return ZooKeeperDB if `use_registry` is `True`, else FallbackDB. + FallbackDB is a compatible class and it actually does nothing. + """ + if not USE_REGISTRY: + return FallbackDB() + if isinstance(USE_REGISTRY, str): + if USE_REGISTRY.lower() == 'zookeeper': + return ZooKeeperDB() + # backward compatibility + return ZooKeeperDB() diff --git a/python/fate_flow/manager/worker_manager.py b/python/fate_flow/manager/service/worker_manager.py similarity index 52% rename from python/fate_flow/manager/worker_manager.py rename to python/fate_flow/manager/service/worker_manager.py index 920920693..a68ad26d3 100644 --- a/python/fate_flow/manager/worker_manager.py +++ b/python/fate_flow/manager/service/worker_manager.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import logging import os +import subprocess import sys from uuid import uuid1 @@ -30,68 +32,55 @@ class WorkerManager: @classmethod - def start_general_worker(cls, worker_name: WorkerName, job_id="", role="", party_id=0, provider=None, - initialized_config: dict = None, run_in_subprocess=True, **kwargs): - pass - - @classmethod - def start_task_worker(cls, worker_name, task: Task, task_parameters, executable: list = None, - extra_env: dict = None, **kwargs): - worker_id, config_dir, log_dir = cls.get_process_dirs( - worker_name=worker_name, - job_id=task.f_job_id, - role=task.f_role, - party_id=task.f_party_id, - task=task) - env = cls.get_env(task.f_job_id, task.f_provider_info, task_parameters) - # config_path, result_path = cls.get_config(config_dir=config_dir, config=task_parameters) - specific_cmd = [] - if worker_name is WorkerName.TASK_EXECUTOR: - from fate_flow.worker.executor import Submit - module_file_path = sys.modules[Submit.__module__].__file__ - else: - raise Exception(f"not support {worker_name} worker") + def start_task_worker(cls, worker_name, task_info, task_parameters=None, executable=None, common_cmd=None, + extra_env: dict = None, record=False, stderr=None, sync=False, **kwargs): + if not extra_env: + extra_env = {} + worker_id = uuid1().hex + config_dir, std_dir = cls.get_process_dirs( + job_id=task_info.get("job_id"), + role=task_info.get("role"), + party_id=task_info.get("party_id"), + task_name=task_info.get("task_name"), + task_version=task_info.get("task_version") + ) + params_env = {} + if task_parameters: + params_env = cls.get_env(task_info.get("job_id"), task_parameters) + extra_env.update(params_env) if executable: process_cmd = executable else: - process_cmd = [env.get("EXECUTOR_ENV") or sys.executable or "python3"] - common_cmd = [ - module_file_path, - "component", - "execute", - "--process-tag", - task.f_execution_id, - "--env-name", - "FATE_TASK_CONFIG", - ] + process_cmd = [os.getenv("EXECUTOR_ENV") or sys.executable or "python3"] process_cmd.extend(common_cmd) - process_cmd.extend(specific_cmd) - p = process_utils.run_subprocess(job_id=task.f_job_id, config_dir=config_dir, process_cmd=process_cmd, - added_env=env, log_dir=log_dir, cwd_dir=config_dir, process_name=worker_name.value, - process_id=worker_id) - cls.save_worker_info(task=task, worker_name=worker_name, worker_id=worker_id, - run_ip=RuntimeConfig.JOB_SERVER_HOST, run_pid=p.pid, config=task_parameters, - cmd=process_cmd) - schedule_logger(job_id=task.f_job_id).info(f"start task worker, executor id {task.f_execution_id}...") - return {"run_pid": p.pid, "run_ip": RuntimeConfig.JOB_SERVER_HOST, "worker_id": worker_id, "cmd": process_cmd} + p = process_utils.run_subprocess(job_id=task_info.get("job_id"), config_dir=config_dir, process_cmd=process_cmd, + added_env=extra_env, std_dir=std_dir, cwd_dir=config_dir, + process_name=worker_name.value, stderr=stderr) + if record: + cls.save_worker_info(task_info=task_info, worker_name=worker_name, worker_id=worker_id, + run_ip=RuntimeConfig.JOB_SERVER_HOST, run_pid=p.pid, config=task_parameters, + cmd=process_cmd) + return { + "run_pid": p.pid, + "run_ip": RuntimeConfig.JOB_SERVER_HOST, + "worker_id": worker_id, + "cmd": process_cmd, + "run_port": RuntimeConfig.HTTP_PORT + } + else: + if sync: + _code = p.wait() + _e = p.stderr.read() if p.stderr else None + if _e and _code: + logging.error(f"process {worker_name.value} run error[code:{_code}]\n: {_e.decode()}") + return p @classmethod - def get_process_dirs(cls, worker_name: WorkerName, worker_id=None, job_id=None, role=None, party_id=None, task: Task = None): - if not worker_id: - worker_id = uuid1().hex - party_id = str(party_id) - if task: - config_dir = job_utils.get_job_directory(job_id, role, party_id, task.f_task_name, task.f_task_id, - str(task.f_task_version), worker_name.value, worker_id) - log_dir = job_utils.get_job_log_directory(job_id, role, party_id, task.f_task_name) - elif job_id and role and party_id: - config_dir = job_utils.get_job_directory(job_id, role, party_id, worker_name.value, worker_id) - log_dir = job_utils.get_job_log_directory(job_id, role, party_id, worker_name.value, worker_id) - else: - config_dir = job_utils.get_general_worker_directory(worker_name.value, worker_id) - log_dir = job_utils.get_general_worker_log_directory(worker_name.value, worker_id) + def get_process_dirs(cls, job_id, role, party_id, task_name, task_version): + config_dir = job_utils.get_job_directory(job_id, role, party_id, task_name, str(task_version)) + std_dir = job_utils.get_job_log_directory(job_id, role, party_id, task_name, "stdout") os.makedirs(config_dir, exist_ok=True) - return worker_id, config_dir, log_dir + return config_dir, std_dir @classmethod def get_config(cls, config_dir, config): @@ -102,15 +91,11 @@ def get_config(cls, config_dir, config): return config_path, result_path @classmethod - def get_env(cls, job_id, provider_info, task_parameters): - # todo: get env by provider + def get_env(cls, job_id, task_parameters): env = { - "PYTHONPATH": os.getenv("PYTHONPATH"), "FATE_JOB_ID": job_id, "FATE_TASK_CONFIG": yaml.dump(task_parameters), } - if os.getenv("EXECUTOR_ENV"): - env["EXECUTOR_ENV"] = os.getenv("EXECUTOR_ENV") return env @classmethod @@ -122,10 +107,11 @@ def cmd_to_func_kwargs(cls, cmd): @classmethod @DB.connection_context() - def save_worker_info(cls, task: Task, worker_name: WorkerName, worker_id, **kwargs): + def save_worker_info(cls, task_info, worker_name: WorkerName, worker_id, **kwargs): worker = WorkerInfo() ignore_attr = auto_date_timestamp_db_field() - for attr, value in task.to_dict().items(): + for attr, value in task_info.items(): + attr = f"f_{attr}" if hasattr(worker, attr) and attr not in ignore_attr and value is not None: setattr(worker, attr, value) worker.f_create_time = current_timestamp() diff --git a/python/fate_flow/operation/base_saver.py b/python/fate_flow/operation/base_saver.py index dcafd0cc3..542782f80 100644 --- a/python/fate_flow/operation/base_saver.py +++ b/python/fate_flow/operation/base_saver.py @@ -15,17 +15,34 @@ # import operator +from functools import reduce +from typing import Type, Union, Dict -from fate_flow.db.base_models import DB, BaseModelOperate +from fate_flow.db.base_models import DB, BaseModelOperate, DataBaseModel from fate_flow.db.db_models import Task, Job from fate_flow.db.schedule_models import ScheduleTask, ScheduleTaskStatus, ScheduleJob -from fate_flow.entity.run_status import JobStatus, TaskStatus, EndStatus +from fate_flow.entity.types import JobStatus, TaskStatus, EndStatus +from fate_flow.errors.server_error import NoFoundJob, NoFoundTask from fate_flow.utils.base_utils import current_timestamp from fate_flow.utils.log_utils import schedule_logger, sql_logger class BaseSaver(BaseModelOperate): STATUS_FIELDS = ["status", "party_status"] + OPERATION = { + '==': operator.eq, + '<': operator.lt, + '<=': operator.le, + '>': operator.gt, + '>=': operator.ge, + '!=': operator.ne, + '<<': operator.lshift, + '>>': operator.rshift, + '%': operator.mod, + '**': operator.pow, + '^': operator.xor, + '~': operator.inv, + } @classmethod def _create_job(cls, job_obj, job_info): @@ -38,7 +55,8 @@ def _create_task(cls, task_obj, task_info): @classmethod @DB.connection_context() def _delete_job(cls, job_obj, job_id): - job_obj.delete().where(job_obj.f_job_id == job_id) + _op = job_obj.delete().where(job_obj.f_job_id == job_id) + return _op.execute() > 0 @classmethod def _update_job_status(cls, job_obj, job_info): @@ -117,6 +135,10 @@ def _update_status(cls, entity_model, entity_info: dict): for status_field in cls.STATUS_FIELDS: if entity_info.get(status_field) and hasattr(entity_model, f"f_{status_field}"): if status_field in ["status", "party_status"]: + # update end time + if hasattr(obj, "f_start_time") and obj.f_start_time: + update_info["end_time"] = current_timestamp() + update_info['elapsed'] = update_info['end_time'] - obj.f_start_time update_info[status_field] = entity_info[status_field] old_status = getattr(obj, f"f_{status_field}") new_status = update_info[status_field] @@ -140,15 +162,24 @@ def _update_status(cls, entity_model, entity_info: dict): @classmethod @DB.connection_context() - def update_entity_table(cls, entity_model, entity_info): + def update_entity_table(cls, entity_model, entity_info, filters: list = None): query_filters = [] primary_keys = entity_model.get_primary_keys_name() - for p_k in primary_keys: - query_filters.append(operator.attrgetter(p_k)(entity_model) == entity_info[p_k.lstrip("f").lstrip("_")]) + if not filters: + for p_k in primary_keys: + query_filters.append(operator.attrgetter(p_k)(entity_model) == entity_info[p_k.lstrip("f").lstrip("_")]) + else: + for _k in filters: + p_k = f"f_{_k}" + query_filters.append(operator.attrgetter(p_k)(entity_model) == entity_info[_k]) objs = entity_model.select().where(*query_filters) if objs: obj = objs[0] else: + if entity_model.__name__ == Job.__name__: + raise NoFoundJob() + if entity_model.__name__ == Job.__name__: + raise NoFoundTask() raise Exception("can not found the {}".format(entity_model.__name__)) update_filters = query_filters[:] update_info = {} @@ -220,4 +251,47 @@ def get_latest_scheduler_tasks(cls, tasks): tasks_group[task.f_task_id] = task elif task.f_task_version > tasks_group[task.f_task_id].f_task_version: tasks_group[task.f_task_id] = task - return tasks_group \ No newline at end of file + return tasks_group + + @classmethod + @DB.connection_context() + def _list(cls, model: Type[DataBaseModel], limit: int = 0, offset: int = 0, + query: dict = None, order_by: Union[str, list, tuple] = None): + data = model.select() + if query: + data = data.where(cls.query_dict2expression(model, query)) + count = data.count() + + if not order_by: + order_by = 'create_time' + if not isinstance(order_by, (list, tuple)): + order_by = (order_by, 'asc') + order_by, order = order_by + order_by = getattr(model, f'f_{order_by}') + order_by = getattr(order_by, order)() + data = data.order_by(order_by) + + if limit > 0: + data = data.limit(limit) + if offset > 0: + data = data.offset(offset) + return list(data), count + + @classmethod + def query_dict2expression(cls, model: Type[DataBaseModel], query: Dict[str, Union[bool, int, str, list, tuple]]): + expression = [] + for field, value in query.items(): + if not isinstance(value, (list, tuple)): + value = ('==', value) + op, *val = value + + field = getattr(model, f'f_{field}') + value = cls.OPERATION[op](field, val[0]) if op in cls.OPERATION else getattr(field, op)(*val) + + expression.append(value) + + return reduce(operator.iand, expression) + + @property + def supported_operators(self): + return \ No newline at end of file diff --git a/python/fate_flow/operation/job_saver.py b/python/fate_flow/operation/job_saver.py index fcddf5c70..fd52e4e09 100644 --- a/python/fate_flow/operation/job_saver.py +++ b/python/fate_flow/operation/job_saver.py @@ -17,6 +17,8 @@ from fate_flow.db.base_models import DB from fate_flow.db.db_models import Job, Task +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundTask from fate_flow.operation.base_saver import BaseSaver from fate_flow.db.schedule_models import ScheduleJob, ScheduleTask, ScheduleTaskStatus @@ -46,10 +48,32 @@ def query_job(cls, reverse=None, order_by=None, **kwargs): def update_job(cls, job_info): return cls._update_job(Job, job_info) + @classmethod + def update_job_user(cls, job_id, user_name): + return cls.update_entity_table(Job, { + "job_id": job_id, + "user_name": user_name + }, filters=["job_id"]) + + @classmethod + def list_job(cls, limit, offset, query, order_by): + return cls._list(Job, limit, offset, query, order_by) + + @classmethod + def list_task(cls, limit, offset, query, order_by): + return cls._list(Task, limit, offset, query, order_by) + @classmethod def query_task(cls, only_latest=True, reverse=None, order_by=None, **kwargs): return cls._query_task(Task, only_latest=only_latest, reverse=reverse, order_by=order_by, **kwargs) + @classmethod + def query_task_by_execution_id(cls, execution_id): + tasks = cls.query_task(execution_id=execution_id) + if not tasks: + raise NoFoundTask(execution_id=execution_id) + return tasks[0] + @classmethod def update_task_status(cls, task_info): return cls._update_task_status(Task, task_info) @@ -104,7 +128,7 @@ def update_task_status(cls, task_info, scheduler_status=False): task_obj = ScheduleTask if scheduler_status: task_obj = ScheduleTaskStatus - cls._update_task_status(task_obj, task_info) + return cls._update_task_status(task_obj, task_info) @classmethod def update_task(cls, task_info, report=False): diff --git a/python/fate_flow/proto/__init__.py b/python/fate_flow/proto/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/proto/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/runtime/component_provider.py b/python/fate_flow/runtime/component_provider.py new file mode 100644 index 000000000..0db4932f5 --- /dev/null +++ b/python/fate_flow/runtime/component_provider.py @@ -0,0 +1,66 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Union + +from fate_flow.entity import BaseEntity +from fate_flow.entity.spec.flow import ProviderSpec, LocalProviderSpec, DockerProviderSpec, K8sProviderSpec + + +class ComponentProvider(BaseEntity): + def __init__(self, provider_info: ProviderSpec): + self._name = provider_info.name + self._device = provider_info.device + self._version = provider_info.version + self._metadata = provider_info.metadata + self._python_path = "" + self._python_env = "" + self.init_env() + + def init_env(self): + if isinstance(self._metadata, LocalProviderSpec): + self._python_path = self._metadata.path + self._python_env = self._metadata.venv + + @property + def name(self): + return self._name + + @property + def device(self): + return self._device + + @property + def version(self): + return self._version + + @property + def metadata(self) -> Union[LocalProviderSpec, DockerProviderSpec, K8sProviderSpec]: + return self._metadata + + @property + def python_path(self): + return self._python_path + + @property + def python_env(self): + return self._python_env + + @property + def provider_name(self): + return f"{self.name}:{self.version}@{self.device}" + + def __eq__(self, other): + return self.name == other.name and self.version == other.version diff --git a/python/fate_flow/runtime/env.py b/python/fate_flow/runtime/env.py new file mode 100644 index 000000000..d66c87751 --- /dev/null +++ b/python/fate_flow/runtime/env.py @@ -0,0 +1,10 @@ +import sys +import fate_flow + + +def is_in_virtualenv(): + try: + module_path = fate_flow.__file__ + return sys.prefix in module_path + except ImportError: + return False diff --git a/python/fate_flow/runtime/job_default_config.py b/python/fate_flow/runtime/job_default_config.py index 2e72d890a..95e7fc929 100644 --- a/python/fate_flow/runtime/job_default_config.py +++ b/python/fate_flow/runtime/job_default_config.py @@ -13,42 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from fate_flow.settings import FATE_FLOW_JOB_DEFAULT_CONFIG_PATH, stat_logger +from fate_flow.runtime.system_settings import FATE_FLOW_JOB_DEFAULT_CONFIG_PATH from .reload_config_base import ReloadConfigBase from ..utils import file_utils +from ..utils.log import getLogger +stat_logger = getLogger() -class JobDefaultConfig(ReloadConfigBase): - # component provider - default_component_provider_path = None - - # Resource - total_cores_overweight_percent = None - total_memory_overweight_percent = None - task_parallelism = None - task_cores = None - task_memory = None - max_cores_percent_per_job = None - # scheduling +class JobDefaultConfig(ReloadConfigBase): + job_cores = None + computing_partitions = None + task_run = None remote_request_timeout = None federated_command_trys = None job_timeout = None - end_status_job_scheduling_time_limit = None - end_status_job_scheduling_updates = None auto_retries = None - auto_retry_delay = None - federated_status_collect_type = None - detect_connect_max_retry_count = None - detect_connect_long_retry_count = None - - # upload - upload_block_max_bytes = None # bytes - - # component output - output_data_summary_count_limit = None + sync_type = None - task_default_conf = None + task_logger = None + task_device = None + launcher = None @classmethod def load(cls): diff --git a/python/fate_flow/runtime/runtime_config.py b/python/fate_flow/runtime/runtime_config.py index f54d91d74..4ebe79e6b 100644 --- a/python/fate_flow/runtime/runtime_config.py +++ b/python/fate_flow/runtime/runtime_config.py @@ -18,15 +18,17 @@ from fate_flow.runtime.reload_config_base import ReloadConfigBase from fate_flow.utils.version import get_versions +from fate_flow.hub.scheduler import JobSchedulerABC class RuntimeConfig(ReloadConfigBase): - DEBUG = None HTTP_PORT = None JOB_SERVER_HOST = None - IS_SERVER = False PROCESS_ROLE = None SCHEDULE_CLIENT: FlowSchedulerApi = None + SCHEDULER: JobSchedulerABC = None + CLIENT_ROLE = list() + SERVICE_DB = None ENV = dict() @classmethod @@ -62,3 +64,17 @@ def set_process_role(cls, process_role: ProcessRole): @classmethod def set_schedule_client(cls, schedule_client): cls.SCHEDULE_CLIENT = schedule_client + + @classmethod + def set_scheduler(cls, scheduler): + cls.SCHEDULER = scheduler + + @classmethod + def set_client_roles(cls, *roles): + for role in roles: + if role not in cls.CLIENT_ROLE: + cls.CLIENT_ROLE.append(role) + + @classmethod + def set_service_db(cls, service_db): + cls.SERVICE_DB = service_db diff --git a/python/fate_flow/runtime/system_settings.py b/python/fate_flow/runtime/system_settings.py new file mode 100644 index 000000000..dd1aaff55 --- /dev/null +++ b/python/fate_flow/runtime/system_settings.py @@ -0,0 +1,153 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from grpc._cython import cygrpc + +from fate_flow.entity.types import ComputingEngine +from fate_flow.runtime.env import is_in_virtualenv +from fate_flow.utils import engine_utils, file_utils +from fate_flow.utils.conf_utils import get_base_config +from fate_flow.utils.file_utils import get_fate_flow_directory, get_fate_python_path + +from fate_flow.settings import * + +# Server +API_VERSION = "v2" +INTERCONN_API_VERSION = "v1" +FATE_FLOW_SERVICE_NAME = "fateflow" +SERVER_MODULE = "fate_flow_server.py" +CASBIN_TABLE_NAME = "fate_casbin" +PERMISSION_TABLE_NAME = "permission_casbin" +PERMISSION_MANAGER_PAGE = "permission" +APP_MANAGER_PAGE = "app" + +ADMIN_PAGE = [PERMISSION_MANAGER_PAGE, APP_MANAGER_PAGE] +FATE_FLOW_CONF_PATH = os.path.join(get_fate_flow_directory(), "conf") + +FATE_FLOW_JOB_DEFAULT_CONFIG_PATH = os.path.join(FATE_FLOW_CONF_PATH, "job_default_config.yaml") + +SUBPROCESS_STD_LOG_NAME = "std.log" + + +HOST = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("host", "127.0.0.1") +HTTP_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("http_port") +GRPC_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("grpc_port") + +NGINX_HOST = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("nginx", {}).get("host") or HOST +NGINX_HTTP_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("nginx", {}).get("http_port") or HTTP_PORT +RANDOM_INSTANCE_ID = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("random_instance_id", False) + +PROTOCOL = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("protocol", "http") + +PROXY_NAME = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("proxy_name") +PROXY_PROTOCOL = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("protocol", "http") +PROXY = get_base_config("federation") +STORAGE = get_base_config("storage") +ENGINES = engine_utils.get_engines() +IS_STANDALONE = engine_utils.is_standalone() +WORKER = get_base_config("worker", {}) +DEFAULT_PROVIDER = get_base_config("default_provider", {}) +CASBIN_MODEL_CONF = os.path.join(FATE_FLOW_CONF_PATH, "casbin_model.conf") +PERMISSION_CASBIN_MODEL_CONF = os.path.join(FATE_FLOW_CONF_PATH, "permission_casbin_model.conf") +SERVICE_CONF_NAME = "service_conf.yaml" + +DATABASE = get_base_config("database", {}) + + +IGNORE_RESOURCE_ROLES = {"arbiter"} + +SUPPORT_IGNORE_RESOURCE_ENGINES = { + ComputingEngine.EGGROLL, ComputingEngine.STANDALONE +} +DEFAULT_FATE_PROVIDER_PATH = (DEFAULT_FATE_DIR or get_fate_python_path()) if not is_in_virtualenv() else "" +HEADERS = { + "Content-Type": "application/json", + "Connection": "close", + "service": FATE_FLOW_SERVICE_NAME +} + +BASE_URI = f"{PROTOCOL}://{HOST}:{HTTP_PORT}/{API_VERSION}" + +HOOK_MODULE = get_base_config("hook_module") +# computing +COMPUTING_CONF = get_base_config("computing", {}) + +# authentication +AUTHENTICATION_CONF = get_base_config("authentication", {}) +# client +CLIENT_AUTHENTICATION = AUTHENTICATION_CONF.get("client", False) +# site +SITE_AUTHENTICATION = AUTHENTICATION_CONF.get("site", False) +# permission +PERMISSION_SWITCH = AUTHENTICATION_CONF.get("permission", False) + +ENCRYPT_CONF = get_base_config("encrypt") + +PARTY_ID = get_base_config("party_id", "") +LOCAL_PARTY_ID = "0" + +MODEL_STORE = get_base_config("model_store") + +GRPC_OPTIONS = [ + (cygrpc.ChannelArgKey.max_send_message_length, -1), + (cygrpc.ChannelArgKey.max_receive_message_length, -1), +] + +LOG_DIR = LOG_DIR or get_fate_flow_directory("logs") +JOB_DIR = JOB_DIR or get_fate_flow_directory("jobs") +MODEL_STORE_PATH = MODEL_DIR or os.path.join(get_fate_flow_directory(), "model") +LOCAL_DATA_STORE_PATH = DATA_DIR or os.path.join(get_fate_flow_directory(), "data") +LOG_LEVEL = LOG_LEVEL or 10 +LOG_SHARE = False +FATE_FLOW_LOG_DIR = os.path.join(LOG_DIR, "fate_flow") +WORKERS_DIR = os.path.join(LOG_DIR, "workers") + +SQLITE_FILE_DIR = SQLITE_FILE_DIR or get_fate_flow_directory() +SQLITE_PATH = os.path.join(SQLITE_FILE_DIR, SQLITE_FILE_NAME) + +GRPC_SERVER_MAX_WORKERS = GRPC_SERVER_MAX_WORKERS or (os.cpu_count() or 1) * 5 + +VERSION_FILE_PATH = os.path.join(get_fate_flow_directory(), "fateflow.env") +FATE_FLOW_PROVIDER_PATH = get_fate_flow_directory("python") +FATE_FLOW_CONF_PATH = get_fate_flow_directory() + +# Registry +FATE_FLOW_MODEL_TRANSFER_ENDPOINT = "/v1/model/transfer" +ZOOKEEPER = get_base_config("zookeeper", {}) +ZOOKEEPER_REGISTRY = { + # server + 'flow-server': "/FATE-COMPONENTS/fate-flow", + # model service + 'fateflow': "/FATE-SERVICES/flow/online/transfer/providers", + 'servings': "/FATE-SERVICES/serving/online/publishLoad/providers", +} +USE_REGISTRY = get_base_config("use_registry") + +REQUEST_TRY_TIMES = 3 +REQUEST_WAIT_SEC = 2 +REQUEST_MAX_WAIT_SEC = 300 + +DEFAULT_OUTPUT_DATA_PARTITIONS = 16 + +STANDALONE_DATA_HOME = os.path.join(file_utils.get_fate_flow_directory(), "data") +LOCALFS_DATA_HOME = os.path.join(file_utils.get_fate_flow_directory(), "localfs") + +# hub module settings +# define： xxx.class_name +DEFAULT_JOB_PARSER_MODULE = "fate_flow.hub.parser.fate.JobParser" +DEFAULT_JOB_SCHEDULER_MODULE = "fate_flow.hub.scheduler.fate.DAGScheduler" +DEFAULT_COMPONENTS_WRAPS_MODULE = "fate_flow.hub.components_wraps.fate.FlowWraps" diff --git a/python/fate_flow/scheduler/__init__.py b/python/fate_flow/scheduler/__init__.py index 482e05d11..05077f7c2 100644 --- a/python/fate_flow/scheduler/__init__.py +++ b/python/fate_flow/scheduler/__init__.py @@ -12,17 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from fate_flow.hub.flow_hub import FlowHub from ofx.api.client import FlowSchedulerApi from fate_flow.runtime.runtime_config import RuntimeConfig -from fate_flow.settings import HOST, HTTP_PORT, PROXY_PROTOCOL, API_VERSION, HTTP_REQUEST_TIMEOUT -from fate_flow.utils.api_utils import get_federated_proxy_address +from fate_flow.runtime.system_settings import HOST, HTTP_PORT, PROXY_PROTOCOL, API_VERSION, HTTP_REQUEST_TIMEOUT +from fate_flow.utils.api_utils import get_federated_proxy_address, generate_headers def init_scheduler(): remote_host, remote_port, remote_protocol, grpc_channel = get_federated_proxy_address() protocol = remote_protocol if remote_protocol else PROXY_PROTOCOL - RuntimeConfig.set_schedule_client(FlowSchedulerApi(host=HOST, port=HTTP_PORT, protocol=protocol, + RuntimeConfig.set_schedule_client(FlowSchedulerApi(host=HOST, port=HTTP_PORT, api_version=API_VERSION, timeout=HTTP_REQUEST_TIMEOUT, remote_protocol=protocol, remote_host=remote_host, - remote_port=remote_port, grpc_channel=grpc_channel)) + remote_port=remote_port, grpc_channel=grpc_channel, + callback=generate_headers)) + + RuntimeConfig.set_scheduler(FlowHub.load_job_scheduler()) diff --git a/python/fate_flow/scheduler/federated_scheduler.py b/python/fate_flow/scheduler/federated_scheduler.py index 26322a2e0..3a76c5c1e 100644 --- a/python/fate_flow/scheduler/federated_scheduler.py +++ b/python/fate_flow/scheduler/federated_scheduler.py @@ -15,8 +15,8 @@ # from functools import wraps -from fate_flow.entity.run_status import FederatedSchedulingStatusCode -from fate_flow.entity.types import ReturnCode +from fate_flow.entity.code import FederatedSchedulingStatusCode +from fate_flow.entity.code import ReturnCode from fate_flow.operation.job_saver import ScheduleJobSaver from fate_flow.runtime.runtime_config import RuntimeConfig from fate_flow.utils.log_utils import schedule_logger @@ -92,8 +92,8 @@ class FederatedScheduler: # Job @classmethod @federated - def create_job(cls, job_id, roles, job_info): - return RuntimeConfig.SCHEDULE_CLIENT.federated.create_job(job_id, roles, command_body=job_info) + def create_job(cls, job_id, roles, initiator_party_id, job_info): + return RuntimeConfig.SCHEDULE_CLIENT.federated.create_job(job_id, roles, initiator_party_id=initiator_party_id, command_body=job_info) @classmethod @federated @@ -120,11 +120,6 @@ def stop_job(cls, job_id, roles): def update_job(cls, job_id, roles, command_body=None): return RuntimeConfig.SCHEDULE_CLIENT.federated.update_job(job_id, roles, command_body) - @classmethod - @federated - def save_pipelined_model(cls, job_id, roles): - return RuntimeConfig.SCHEDULE_CLIENT.federated.save_pipelined_model(job_id, roles) - # task @classmethod @federated_task @@ -164,8 +159,8 @@ def rerun_task(cls, task_id, task_version, tasks=None): # scheduler @classmethod @schedule_job - def request_create_job(cls, party_id, command_body): - return RuntimeConfig.SCHEDULE_CLIENT.scheduler.create_job(party_id, command_body) + def request_create_job(cls, party_id, initiator_party_id, command_body): + return RuntimeConfig.SCHEDULE_CLIENT.scheduler.create_job(party_id, initiator_party_id, command_body) @classmethod @schedule_job diff --git a/python/fate_flow/scheduler/job_scheduler.py b/python/fate_flow/scheduler/job_scheduler.py index 4a242a95c..1e48c9ea3 100644 --- a/python/fate_flow/scheduler/job_scheduler.py +++ b/python/fate_flow/scheduler/job_scheduler.py @@ -15,383 +15,23 @@ # from pydantic import typing -from fate_flow.controller.task_controller import TaskController -from fate_flow.entity.dag_structures import DAGSchema -from fate_flow.hub.flow_hub import FlowHub -from fate_flow.scheduler.task_scheduler import TaskScheduler -from fate_flow.db.base_models import DB -from fate_flow.db.schedule_models import ScheduleJob, ScheduleTaskStatus -from fate_flow.entity.run_status import StatusSet, FederatedSchedulingStatusCode, JobStatus, TaskStatus, EndStatus, \ - SchedulingStatusCode, InterruptStatus -from fate_flow.entity.types import ResourceOperation, ReturnCode -from fate_flow.operation.job_saver import ScheduleJobSaver, JobSaver -from fate_flow.runtime.job_default_config import JobDefaultConfig -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.utils import job_utils, schedule_utils -from fate_flow.utils.base_utils import current_timestamp, json_dumps +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.db.schedule_models import ScheduleTaskStatus from fate_flow.utils.cron import Cron -from fate_flow.utils.log_utils import schedule_logger, exception_to_trace_string class DAGScheduler(Cron): @classmethod - def submit(cls, dag_schema: DAGSchema): - job_id = job_utils.generate_job_id() - schedule_logger(job_id).info(f"submit job, dag {dag_schema.dag.dict()}, schema version {dag_schema.schema_version}") - submit_result = { - "job_id": job_id, - "data": {} - } - try: - job = ScheduleJob() - job.f_job_id = job_id - job.f_parties = [party.dict() for party in dag_schema.dag.parties] - job.f_initiator_party_id = dag_schema.dag.conf.initiator_party_id - job.f_scheduler_party_id = dag_schema.dag.conf.scheduler_party_id - cls.fill_default_job_parameters(job_id, dag_schema) - job.f_dag = dag_schema.dict() - submit_result["data"].update({ - "model_id": dag_schema.dag.conf.model_id, - "model_version": dag_schema.dag.conf.model_version - }) - job.f_status = StatusSet.READY - ScheduleJobSaver.create_job(job.to_human_model_dict()) - status_code, response = FederatedScheduler.create_job( - job_id, job.f_parties, {"dag_schema": dag_schema.dict(), "job_id": job_id} - ) - if status_code != FederatedSchedulingStatusCode.SUCCESS: - job.f_status = JobStatus.FAILED - FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, job_info={ - "job_id": job.f_job_id, - "status": job.f_status - }) - raise Exception("create job failed", response) - else: - job.f_status = JobStatus.WAITING - TaskController.create_schedule_tasks(job, dag_schema) - status_code, response = FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, - job_info={"job_id": job.f_job_id, - "status": job.f_status}) - if status_code != FederatedSchedulingStatusCode.SUCCESS: - raise Exception(f"set job to waiting status failed: {response}") - ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": job.f_status}) - schedule_logger(job_id).info(f"submit job successfully, job id is {job.f_job_id}") - result = { - "code": ReturnCode.Base.SUCCESS, - "message": "success" - } - submit_result.update(result) - except Exception as e: - schedule_logger(job_id).exception(e) - submit_result["code"] = ReturnCode.Job.CREATE_JOB_FAILED - submit_result["message"] = exception_to_trace_string(e) - return submit_result - - @classmethod - def fill_default_job_parameters(cls, job_id: str, dag_schema: DAGSchema): - if not dag_schema.dag.conf.federated_status_collect_type: - dag_schema.dag.conf.federated_status_collect_type = JobDefaultConfig.federated_status_collect_type - if not dag_schema.dag.conf.model_id or not dag_schema.dag.conf.model_id: - dag_schema.dag.conf.model_id, dag_schema.dag.conf.model_version = job_utils.generate_model_info(job_id) - if not dag_schema.dag.conf.auto_retries: - dag_schema.dag.conf.auto_retries = JobDefaultConfig.auto_retries + def submit(cls, dag_schema): + return RuntimeConfig.SCHEDULER.submit(dag_schema) def run_do(self): - # waiting - schedule_logger().info("start schedule waiting jobs") - jobs = ScheduleJobSaver.query_job(status=JobStatus.WAITING, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} waiting jobs") - if len(jobs): - job = jobs[0] - schedule_logger().info(f"schedule waiting job {job.f_job_id}") - try: - self.schedule_waiting_jobs(job=job) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule waiting job failed") - schedule_logger().info("schedule waiting jobs finished") - - # running - schedule_logger().info("start schedule running jobs") - jobs = ScheduleJobSaver.query_job(status=JobStatus.RUNNING, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} running jobs") - for job in jobs: - schedule_logger().info(f"schedule running job {job.f_job_id}") - try: - self.schedule_running_job(job=job) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule job failed") - schedule_logger().info("schedule running jobs finished") - - # ready - schedule_logger().info("start schedule ready jobs") - jobs = ScheduleJobSaver.query_job(ready_signal=True, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} ready jobs") - for job in jobs: - schedule_logger().info(f"schedule ready job {job.f_job_id}") - try: - pass - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error(f"schedule ready job failed:\n{e}") - schedule_logger().info("schedule ready jobs finished") - - # rerun - schedule_logger().info("start schedule rerun jobs") - jobs = ScheduleJobSaver.query_job(rerun_signal=True, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} rerun jobs") - for job in jobs: - schedule_logger(job.f_job_id).info(f"schedule rerun job {job.f_job_id}") - try: - self.schedule_rerun_job(job=job) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule job failed") - schedule_logger().info("schedule rerun jobs finished") - - # end - schedule_logger().info("start schedule end status jobs to update status") - jobs = ScheduleJobSaver.query_job(status=set(EndStatus.status_list()), - end_time=[current_timestamp() - JobDefaultConfig.end_status_job_scheduling_time_limit, - current_timestamp()]) - schedule_logger().info(f"have {len(jobs)} end status jobs") - for job in jobs: - schedule_logger().info(f"schedule end status job {job.f_job_id}") - try: - update_status = self.end_scheduling_updates(job_id=job.f_job_id) - if update_status: - schedule_logger(job.f_job_id).info("try update status by scheduling like running job") - else: - schedule_logger(job.f_job_id).info("the number of updates has been exceeded") - continue - self.schedule_running_job(job=job, force_sync_status=True) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule job failed") - schedule_logger().info("schedule end status jobs finished") - - @classmethod - def apply_job_resource(cls, job): - apply_status_code, federated_response = FederatedScheduler.resource_for_job( - job_id=job.f_job_id, - roles=job.f_parties, - operation_type=ResourceOperation.APPLY.value - ) - if apply_status_code == FederatedSchedulingStatusCode.SUCCESS: - return True - else: - # rollback resource - rollback_party = [] - failed_party = [] - for dest_role in federated_response.keys(): - for dest_party_id in federated_response[dest_role].keys(): - retcode = federated_response[dest_role][dest_party_id]["code"] - if retcode == ReturnCode.Base.SUCCESS: - rollback_party.append({"role": dest_role, "party_id": [dest_party_id]}) - else: - failed_party.append({"role": dest_role, "party_id": [dest_party_id]}) - schedule_logger(job.f_job_id).info("job apply resource failed on {}, rollback {}".format(failed_party, - rollback_party)) - if rollback_party: - return_status_code, federated_response = FederatedScheduler.resource_for_job( - job_id=job.f_job_id, - roles=rollback_party, - operation_type=ResourceOperation.RETURN.value - ) - if return_status_code != FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info(f"job return resource failed:\n{federated_response}") - else: - schedule_logger(job.f_job_id).info("job no party should be rollback resource") - return False - - @classmethod - def schedule_waiting_jobs(cls, job: ScheduleJob): - job_id = job.f_job_id - if job.f_cancel_signal: - FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, - job_info={"job_id": job.f_job_id, "status": JobStatus.CANCELED}) - ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": JobStatus.CANCELED}) - schedule_logger(job.f_job_id).info("job have cancel signal") - return - status = cls.apply_job_resource(job) - if status: - cls.start_job(job_id=job.f_job_id, roles=job.f_parties) - - def schedule_running_job(self, job: ScheduleJob, force_sync_status=False): - schedule_logger(job.f_job_id).info("scheduling running job") - job_parser = FlowHub.load_job_parser(DAGSchema(**job.f_dag)) - task_scheduling_status_code, auto_rerun_tasks, tasks = TaskScheduler.schedule(job=job, job_parser=job_parser, - canceled=job.f_cancel_signal, - dag_schema=DAGSchema(**job.f_dag)) - tasks_status = dict([(task.f_task_name, task.f_status) for task in tasks]) - schedule_logger(job_id=job.f_job_id).info(f"task_scheduling_status_code: {task_scheduling_status_code}, " - f"tasks_status: {tasks_status.values()}") - new_job_status = self.calculate_job_status(task_scheduling_status_code=task_scheduling_status_code, tasks_status=tasks_status.values()) - if new_job_status == JobStatus.WAITING and job.f_cancel_signal: - new_job_status = JobStatus.CANCELED - total, finished_count = self.calculate_job_progress(tasks_status=tasks_status) - new_progress = float(finished_count) / total * 100 - schedule_logger(job.f_job_id).info(f"job status is {new_job_status}, calculate by task status list: {tasks_status}") - if new_job_status != job.f_status or new_progress != job.f_progress: - # Make sure to update separately, because these two fields update with anti-weight logic - if int(new_progress) - job.f_progress > 0: - job.f_progress = new_progress - FederatedScheduler.update_job(job_id=job.f_job_id, - roles=job.f_parties, - command_body={"job_id": job.f_job_id, "progress": job.f_progress}) - self.update_job_on_scheduler(schedule_job=job, update_fields=["progress"]) - if new_job_status != job.f_status: - job.f_status = new_job_status - if EndStatus.contains(job.f_status): - FederatedScheduler.save_pipelined_model(job_id=job.f_job_id, roles=job.f_parties) - FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, - job_info={"job_id": job.f_job_id, - "status": job.f_status}) - self.update_job_on_scheduler(schedule_job=job, update_fields=["status"]) - if EndStatus.contains(job.f_status): - self.finish(job=job, end_status=job.f_status) - if auto_rerun_tasks: - schedule_logger(job.f_job_id).info("job have auto rerun tasks") - self.set_job_rerun(job_id=job.f_job_id, tasks=auto_rerun_tasks, auto=True) - if force_sync_status: - FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_roles, status=job.f_status, - job_info=job.to_human_model_dict()) - schedule_logger(job.f_job_id).info("finish scheduling running job") - - def schedule_rerun_job(self, job): - if EndStatus.contains(job.f_status): - job.f_status = JobStatus.WAITING - schedule_logger(job.f_job_id).info("job has been finished, set waiting to rerun") - status, response = FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, - job_info={"job_id": job.f_job_id, - "status": job.f_status}) - if status == FederatedSchedulingStatusCode.SUCCESS: - schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) - schedule_logger(job.f_job_id).info("job set waiting to rerun successfully") - ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": job.f_status}) - else: - schedule_logger(job.f_job_id).info("job set waiting to rerun failed") - else: - schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) - self.schedule_running_job(job) - - @classmethod - def calculate_job_status(cls, task_scheduling_status_code, tasks_status): - tmp_status_set = set(tasks_status) - if TaskStatus.PASS in tmp_status_set: - tmp_status_set.remove(TaskStatus.PASS) - tmp_status_set.add(TaskStatus.SUCCESS) - if len(tmp_status_set) == 1: - return tmp_status_set.pop() - else: - if TaskStatus.RUNNING in tmp_status_set: - return JobStatus.RUNNING - if TaskStatus.WAITING in tmp_status_set: - if task_scheduling_status_code == SchedulingStatusCode.HAVE_NEXT: - return JobStatus.RUNNING - else: - pass - for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), reverse=True): - if status in tmp_status_set: - return status - if tmp_status_set == {TaskStatus.WAITING, TaskStatus.SUCCESS} and task_scheduling_status_code == SchedulingStatusCode.NO_NEXT: - return JobStatus.CANCELED - - raise Exception("calculate job status failed, all task status: {}".format(tasks_status)) - - @classmethod - def calculate_job_progress(cls, tasks_status): - total = 0 - finished_count = 0 - for task_status in tasks_status.values(): - total += 1 - if EndStatus.contains(task_status): - finished_count += 1 - return total, finished_count - - @classmethod - def start_job(cls, job_id, roles): - schedule_logger(job_id).info(f"start job {job_id}") - status_code, response = FederatedScheduler.start_job(job_id, roles) - schedule_logger(job_id).info(f"start job {job_id} status code: {status_code}, response: {response}") - ScheduleJobSaver.update_job_status(job_info={"job_id": job_id, "status": StatusSet.RUNNING}) + return RuntimeConfig.SCHEDULER.run_do() @classmethod def stop_job(cls, job_id, stop_status): - schedule_logger(job_id).info(f"request stop job with {stop_status}") - jobs = ScheduleJobSaver.query_job(job_id=job_id) - if len(jobs) > 0: - if stop_status == JobStatus.CANCELED: - schedule_logger(job_id).info("cancel job") - set_cancel_status = schedule_utils.cancel_signal(job_id=job_id, set_or_reset=True) - schedule_logger(job_id).info(f"set job cancel signal {set_cancel_status}") - job = jobs[0] - job.f_status = stop_status - schedule_logger(job_id).info(f"request stop job with {stop_status} to all party") - status_code, response = FederatedScheduler.stop_job(job_id=job_id, roles=job.f_parties) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job_id).info(f"stop job with {stop_status} successfully") - return ReturnCode.Base.SUCCESS, "success" - else: - tasks_group = ScheduleJobSaver.get_status_tasks_asc(job_id=job.f_job_id) - for task in tasks_group.values(): - TaskScheduler.collect_task_of_all_party(job, task=task, set_status=stop_status) - schedule_logger(job_id).info(f"stop job with {stop_status} failed, {response}") - return ReturnCode.Job.KILL_FAILED, json_dumps(response) - else: - return ReturnCode.Job.NOT_FOUND, "job not found" - - @classmethod - @DB.connection_context() - def end_scheduling_updates(cls, job_id): - operate = ScheduleJob.update({ - ScheduleJob.f_end_scheduling_updates: ScheduleJob.f_end_scheduling_updates + 1} - ).where( - ScheduleJob.f_job_id == job_id, - ScheduleJob.f_end_scheduling_updates < JobDefaultConfig.end_status_job_scheduling_updates - ) - update_status = operate.execute() > 0 - return update_status - - @classmethod - def update_job_on_scheduler(cls, schedule_job: ScheduleJob, update_fields: list): - schedule_logger(schedule_job.f_job_id).info(f"try to update job {update_fields} on scheduler") - jobs = ScheduleJobSaver.query_job(job_id=schedule_job.f_job_id) - if not jobs: - raise Exception("Failed to update job status on scheduler") - job_info = schedule_job.to_human_model_dict(only_primary_with=update_fields) - for field in update_fields: - job_info[field] = getattr(schedule_job, "f_%s" % field) - if "status" in update_fields: - ScheduleJobSaver.update_job_status(job_info=job_info) - ScheduleJobSaver.update_job(job_info=job_info) - schedule_logger(schedule_job.f_job_id).info(f"update job {update_fields} on scheduler finished") - - @classmethod - def set_job_rerun(cls, job_id, auto, force=False, tasks: typing.List[ScheduleTaskStatus] = None, - component_name: typing.Union[str, list] = None): - schedule_logger(job_id).info(f"try to rerun job {job_id}") - jobs = ScheduleJobSaver.query_job(job_id=job_id) - if not jobs: - raise RuntimeError(f"can not found job {job_id}") - job = jobs[0] - if tasks: - schedule_logger(job_id).info(f"require {[task.f_task_name for task in tasks]} to rerun") - else: - # todo: get_need_revisit_nodes - tasks = ScheduleJobSaver.query_task(job_id=job_id, status=TaskStatus.CANCELED, scheduler_status=True) - job_can_rerun = any([TaskController.prepare_rerun_task( - job=job, task=task, auto=auto, force=force, - ) for task in tasks]) - schedule_logger(job_id).info("job set rerun signal") - status = schedule_utils.rerun_signal(job_id=job_id, set_or_reset=True) - schedule_logger(job_id).info(f"job set rerun signal {'successfully' if status else 'failed'}") - return True + return RuntimeConfig.SCHEDULER.stop_job(job_id, stop_status) @classmethod - def finish(cls, job, end_status): - schedule_logger(job.f_job_id).info(f"job finished with {end_status}, do something...") - cls.stop_job(job_id=job.f_job_id, stop_status=end_status) - # todo: clean job - schedule_logger(job.f_job_id).info(f"job finished with {end_status}, done") + def rerun_job(cls, job_id, auto, tasks: typing.List[ScheduleTaskStatus] = None): + return RuntimeConfig.SCHEDULER.rerun_job(job_id, auto, tasks) diff --git a/python/fate_flow/scheduler/task_scheduler.py b/python/fate_flow/scheduler/task_scheduler.py deleted file mode 100644 index f5e597bce..000000000 --- a/python/fate_flow/scheduler/task_scheduler.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.entity.dag_structures import DAGSchema -from fate_flow.entity.engine_types import FederatedCommunicationType -from fate_flow.entity.types import ReturnCode, ResourceOperation -from fate_flow.entity.run_status import StatusSet, TaskStatus, InterruptStatus, EndStatus, AutoRerunStatus, \ - SchedulingStatusCode -from fate_flow.entity.run_status import FederatedSchedulingStatusCode -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.operation.job_saver import ScheduleJobSaver -from fate_flow.utils.log_utils import schedule_logger - - -class TaskScheduler(object): - @classmethod - def schedule(cls, job, job_parser, dag_schema: DAGSchema, canceled=False): - schedule_logger(job.f_job_id).info("scheduling job tasks") - tasks_group = ScheduleJobSaver.get_status_tasks_asc(job_id=job.f_job_id) - waiting_tasks = {} - auto_rerun_tasks = [] - job_interrupt = False - for task in tasks_group.values(): - if dag_schema.dag.conf.federated_status_collect_type == FederatedCommunicationType.PULL: - cls.collect_task_of_all_party(job=job, task=task) - else: - pass - new_task_status = cls.get_federated_task_status(job_id=task.f_job_id, task_id=task.f_task_id, - task_version=task.f_task_version) - task_interrupt = False - task_status_have_update = False - if new_task_status != task.f_status: - task_status_have_update = True - schedule_logger(job.f_job_id).info(f"sync task status {task.f_status} to {new_task_status}") - task.f_status = new_task_status - FederatedScheduler.sync_task_status(task_id=task.f_task_id, command_body={"status": task.f_status}) - ScheduleJobSaver.update_task_status(task.to_human_model_dict(), scheduler_status=True) - if InterruptStatus.contains(new_task_status): - task_interrupt = True - job_interrupt = True - if task.f_status == TaskStatus.WAITING: - waiting_tasks[task.f_task_name] = task - elif task_status_have_update and EndStatus.contains(task.f_status) or task_interrupt: - schedule_logger(task.f_job_id).info(f"stop task with status: {task.f_status}") - FederatedScheduler.stop_task(task_id=task.f_task_id, command_body={"status": task.f_status}) - if not canceled and AutoRerunStatus.contains(task.f_status): - if task.f_auto_retries > 0: - auto_rerun_tasks.append(task) - schedule_logger(job.f_job_id).info(f"task {task.f_task_id} {task.f_status} will be retried") - else: - schedule_logger(job.f_job_id).info(f"task {task.f_task_id} {task.f_status} has no retry count") - - scheduling_status_code = SchedulingStatusCode.NO_NEXT - schedule_logger(job.f_job_id).info(f"canceled status {canceled}, job interrupt status {job_interrupt}") - if not canceled and not job_interrupt: - for task_id, waiting_task in waiting_tasks.items(): - dependent_tasks = job_parser.infer_dependent_tasks( - dag_schema.dag.tasks[waiting_task.f_task_name].inputs - ) - schedule_logger(job.f_job_id).info(f"task {waiting_task.f_task_name} dependent tasks:{dependent_tasks}") - for task_name in dependent_tasks: - dependent_task = tasks_group[task_name] - if dependent_task.f_status != TaskStatus.SUCCESS: - break - else: - scheduling_status_code = SchedulingStatusCode.HAVE_NEXT - status_code = cls.start_task(job=job, task=waiting_task) - if status_code == SchedulingStatusCode.NO_RESOURCE: - schedule_logger(job.f_job_id).info(f"task {waiting_task.f_task_id} can not apply resource, wait for the next round of scheduling") - break - elif status_code == SchedulingStatusCode.FAILED: - schedule_logger(job.f_job_id).info(f"task status code: {status_code}") - scheduling_status_code = SchedulingStatusCode.FAILED - waiting_task.f_status = StatusSet.FAILED - FederatedScheduler.sync_task_status(task_id=waiting_task.f_task_id, command_body={ - "status": waiting_task.f_status}) - break - else: - schedule_logger(job.f_job_id).info("have cancel signal, pass start job tasks") - schedule_logger(job.f_job_id).info("finish scheduling job tasks") - return scheduling_status_code, auto_rerun_tasks, tasks_group.values() - - @classmethod - def start_task(cls, job, task): - schedule_logger(task.f_job_id).info("try to start task {} {}".format(task.f_task_id, task.f_task_version)) - # apply resource for task - apply_status = cls.apply_task_resource(task, job) - if not apply_status: - return SchedulingStatusCode.NO_RESOURCE - task.f_status = TaskStatus.RUNNING - ScheduleJobSaver.update_task_status( - task_info=task.to_human_model_dict(only_primary_with=["status"]), scheduler_status=True - ) - schedule_logger(task.f_job_id).info("start task {} {}".format(task.f_task_id, task.f_task_version)) - FederatedScheduler.sync_task_status(task_id=task.f_task_id, command_body={"status": task.f_status}) - ScheduleJobSaver.update_task_status(task.to_human_model_dict(), scheduler_status=True) - status_code, response = FederatedScheduler.start_task(task_id=task.f_task_id) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - return SchedulingStatusCode.SUCCESS - else: - return SchedulingStatusCode.FAILED - - @classmethod - def apply_task_resource(cls, task, job): - apply_status_code, federated_response = FederatedScheduler.resource_for_task( - task_id=task.f_task_id, - operation_type=ResourceOperation.APPLY.value - ) - if apply_status_code == FederatedSchedulingStatusCode.SUCCESS: - return True - else: - # rollback resource - rollback_party = [] - failed_party = [] - for dest_role in federated_response.keys(): - for dest_party_id in federated_response[dest_role].keys(): - retcode = federated_response[dest_role][dest_party_id]["code"] - if retcode == ReturnCode.Base.SUCCESS: - rollback_party.append({"role": dest_role, "party_id": [dest_party_id]}) - else: - failed_party.append({"role": dest_role, "party_id": [dest_party_id]}) - schedule_logger(job.f_job_id).info("task apply resource failed on {}, rollback {}".format(failed_party, - rollback_party)) - if rollback_party: - return_status_code, federated_response = FederatedScheduler.resource_for_task( - task_id=task.f_task_id, - roles=rollback_party, - operation_type=ResourceOperation.RETURN.value - ) - if return_status_code != FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info(f"task return resource failed:\n{federated_response}") - else: - schedule_logger(job.f_job_id).info("task no party should be rollback resource") - return False - - @classmethod - def collect_task_of_all_party(cls, job, task, set_status=None): - tasks_on_all_party = ScheduleJobSaver.query_task(task_id=task.f_task_id, task_version=task.f_task_version) - tasks_status_on_all = set([task.f_status for task in tasks_on_all_party]) - if not len(tasks_status_on_all) > 1 and TaskStatus.RUNNING not in tasks_status_on_all: - return - status, federated_response = FederatedScheduler.collect_task(task_id=task.f_task_id) - if status != FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).warning(f"collect task {task.f_task_id} {task.f_task_version} failed") - for _role in federated_response.keys(): - for _party_id, party_response in federated_response[_role].items(): - if party_response["code"] == ReturnCode.Base.SUCCESS: - ScheduleJobSaver.update_task_status(task_info=party_response["data"]) - elif set_status: - tmp_task_info = { - "job_id": task.f_job_id, - "task_id": task.f_task_id, - "task_version": task.f_task_version, - "role": _role, - "party_id": _party_id, - "party_status": set_status - } - ScheduleJobSaver.update_task_status(task_info=tmp_task_info) - - @classmethod - def get_federated_task_status(cls, job_id, task_id, task_version): - tasks_on_all_party = ScheduleJobSaver.query_task(task_id=task_id, task_version=task_version) - tasks_party_status = [task.f_status for task in tasks_on_all_party] - status = cls.calculate_multi_party_task_status(tasks_party_status) - schedule_logger(job_id=job_id).info("task {} {} status is {}, calculate by task party status list: {}".format(task_id, task_version, status, tasks_party_status)) - return status - - @classmethod - def calculate_multi_party_task_status(cls, tasks_party_status): - tmp_status_set = set(tasks_party_status) - if TaskStatus.PASS in tmp_status_set: - tmp_status_set.remove(TaskStatus.PASS) - tmp_status_set.add(TaskStatus.SUCCESS) - if len(tmp_status_set) == 1: - return tmp_status_set.pop() - else: - for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), reverse=True): - if status in tmp_status_set: - return status - if TaskStatus.RUNNING in tmp_status_set: - return TaskStatus.RUNNING - if TaskStatus.SUCCESS in tmp_status_set: - return TaskStatus.RUNNING - raise Exception("Calculate task status failed: {}".format(tasks_party_status)) diff --git a/python/fate_flow/settings.py b/python/fate_flow/settings.py index 00eb81f1e..0839591df 100644 --- a/python/fate_flow/settings.py +++ b/python/fate_flow/settings.py @@ -13,73 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os - -from fate_flow.entity.engine_types import ComputingEngine -from fate_flow.utils import engine_utils -from fate_flow.utils.conf_utils import get_base_config, decrypt_database_config -from fate_flow.utils.file_utils import get_fate_flow_directory -from fate_flow.utils.log_utils import LoggerFactory, getLogger - -# Server -API_VERSION = "v2" -FATE_FLOW_SERVICE_NAME = "fateflow" -SERVER_MODULE = "fate_flow_server.py" -TEMP_DIRECTORY = os.path.join(get_fate_flow_directory(), "temp") -FATE_FLOW_CONF_PATH = os.path.join(get_fate_flow_directory(), "conf") - -FATE_FLOW_JOB_DEFAULT_CONFIG_PATH = os.path.join(FATE_FLOW_CONF_PATH, "job_default_config.yaml") - -SUBPROCESS_STD_LOG_NAME = "std.log" - -GRPC_SERVER_MAX_WORKERS = None +# GRPC +GRPC_SERVER_MAX_WORKERS = None # default: (os.cpu_count() or 1) * 5 +# Request HTTP_REQUEST_TIMEOUT = 10 # s - REMOTE_REQUEST_TIMEOUT = 30000 # ms -HOST = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("host", "127.0.0.1") -HTTP_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("http_port") -GRPC_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("grpc_port") - -PROTOCOL = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("protocol", "http") - -PROXY_NAME = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("proxy_name") -PROXY_PROTOCOL = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("protocol", "http") -PROXY = get_base_config("federation") -FORCE_USE_SQLITE = get_base_config("force_use_sqlite") -ENGINES = engine_utils.get_engines() -IS_STANDALONE = engine_utils.is_standalone() -WORKER = get_base_config("worker", {}) - -DATABASE = decrypt_database_config() - -# Logger -LOG_DIRECTORY = get_fate_flow_directory("logs") -LoggerFactory.set_directory(os.path.join(LOG_DIRECTORY, "fate_flow")) -# {CRITICAL: 50, FATAL:50, ERROR:40, WARNING:30, WARN:30, INFO:20, DEBUG:10, NOTSET:0} -LoggerFactory.LEVEL = 10 - -IGNORE_RESOURCE_ROLES = {"arbiter"} - -SUPPORT_IGNORE_RESOURCE_ENGINES = { - ComputingEngine.EGGROLL, ComputingEngine.STANDALONE -} - -HEADERS = { - "Content-Type": "application/json", - "Connection": "close", - "service": FATE_FLOW_SERVICE_NAME -} +LOG_LEVEL = 20 +LOG_DIR = "" +DATA_DIR = "" +MODEL_DIR = "" +JOB_DIR = "" +DEFAULT_FATE_DIR = "" -stat_logger = getLogger("fate_flow_stat") -detect_logger = getLogger("fate_flow_detect") -access_logger = getLogger("fate_flow_access") -database_logger = getLogger("fate_flow_database") +# sqlite +SQLITE_FILE_DIR = "" +SQLITE_FILE_NAME = "fate_flow_sqlite.db" -PARTY_ID = get_base_config("party_id", "") -SOURCE_MODEL_STORE_PATH = os.path.join(get_fate_flow_directory(), "model", "source") -CACHE_MODEL_STORE_PATH = os.path.join(get_fate_flow_directory(), "model", "cache") -LOCAL_DATA_STORE_PATH = os.path.join(get_fate_flow_directory(), "data") -BASE_URI = f"{PROTOCOL}://{HOST}:{HTTP_PORT}/{API_VERSION}" +# Client Manager +APP_TOKEN_LENGTH = 16 +ADMIN_ID = "admin" +ADMIN_KEY = "fate_flow_admin" diff --git a/python/fate_flow/utils/__init__.py b/python/fate_flow/utils/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/utils/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/utils/api_utils.py b/python/fate_flow/utils/api_utils.py index f1ce5facd..025aed286 100644 --- a/python/fate_flow/utils/api_utils.py +++ b/python/fate_flow/utils/api_utils.py @@ -13,89 +13,114 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import json - -from flask import ( - Response, jsonify -) -from webargs import fields -from webargs.flaskparser import use_kwargs -from werkzeug.http import HTTP_STATUS_CODES - -from fate_flow.entity.engine_types import CoordinationProxyService -from fate_flow.entity.types import CoordinationCommunicationProtocol, FederatedMode, ReturnCode -from fate_flow.settings import stat_logger, PROXY_NAME, ENGINES, PROXY, HOST, HTTP_PORT - - -def get_json_result(code=ReturnCode.Base.SUCCESS, message='success', data=None, job_id=None, meta=None): - result_dict = { - "code": code, - "message": message, - "data": data, - "job_id": job_id, - "meta": meta, - } - - response = {} - for key, value in result_dict.items(): - if value is not None: - response[key] = value - return jsonify(response) - - -def server_error_response(e): - stat_logger.exception(e) - if len(e.args) > 1: - return get_json_result(code=ReturnCode.Base.EXCEPTION_ERROR, message=repr(e.args[0]), data=e.args[1]) - return get_json_result(code=ReturnCode.Base.EXCEPTION_ERROR, message=repr(e)) - - -def args_error_response(e): - stat_logger.exception(e) - messages = e.data.get("messages", {}) - return get_json_result(code=ReturnCode.Base.EXCEPTION_ERROR, message="Invalid request.", data=messages) - - -def error_response(response_code, retmsg=None): - if retmsg is None: - retmsg = HTTP_STATUS_CODES.get(response_code, 'Unknown Error') - - return Response(json.dumps({ - 'retmsg': retmsg, - 'retcode': response_code, - }), status=response_code, mimetype='application/json') - - -def validate_request_json(**kwargs): - return use_kwargs(kwargs, location='json') - - -def validate_request_params(**kwargs): - return use_kwargs(kwargs, location='querystring') - - -def job_request_json(**kwargs): - return validate_request_json( - job_id=fields.String(required=True), - role=fields.String(required=True), - party_id=fields.String(required=True), - **kwargs - ) - - -def task_request_json(**kwargs): - return validate_request_json( - job_id=fields.String(required=True), - task_id=fields.String(required=True), - task_version=fields.Integer(required=True), - role=fields.String(required=True), - party_id=fields.String(required=True), - **kwargs - ) - - -def validate_request_headers(**kwargs): - return use_kwargs(kwargs, location='headers') +import random +import time +from functools import wraps + +import marshmallow +from flask import jsonify, send_file, request as flask_request + +from webargs.flaskparser import parser + +from fate_flow.entity.types import CoordinationProxyService, CoordinationCommunicationProtocol, FederatedMode +from fate_flow.entity.code import ReturnCode +from fate_flow.errors import FateFlowError +from fate_flow.hook import HookManager +from fate_flow.hook.common.parameters import SignatureParameters +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.runtime.system_settings import PROXY_NAME, ENGINES, PROXY, HOST, HTTP_PORT, API_VERSION, \ + REQUEST_TRY_TIMES, REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC +from fate_flow.utils.log import getLogger +from fate_flow.utils.log_utils import schedule_logger, audit_logger +from fate_flow.utils.requests_utils import request + +parser.unknown = marshmallow.EXCLUDE + +stat_logger = getLogger() + + +class API: + class Input: + @staticmethod + def params(**kwargs): + return parser.use_kwargs(kwargs, location='querystring') + + @staticmethod + def form(**kwargs): + return parser.use_kwargs(kwargs, location='form') + + @staticmethod + def files(**kwargs): + return parser.use_kwargs(kwargs, location='files') + + @staticmethod + def json(**kwargs): + return parser.use_kwargs(kwargs, location='json') + + @staticmethod + def headers(**kwargs): + return parser.use_kwargs(kwargs, location="headers") + + class Output: + @staticmethod + def json(code=ReturnCode.Base.SUCCESS, message='success', data=None, job_id=None, **kwargs): + result_dict = { + "code": code, + "message": message, + "data": data, + "job_id": job_id, + } + + response = {} + for key, value in result_dict.items(): + if value is not None: + response[key] = value + # extra resp + for key, value in kwargs.items(): + response[key] = value + return jsonify(response) + + @staticmethod + def file(path_or_file, attachment_filename, as_attachment, mimetype="application/octet-stream"): + return send_file(path_or_file, download_name=attachment_filename, as_attachment=as_attachment, mimetype=mimetype) + + @staticmethod + def server_error_response(e): + if isinstance(e, FateFlowError): + return API.Output.json(code=e.code, message=e.message) + stat_logger.exception(e) + if len(e.args) > 1: + if isinstance(e.args[0], int): + return API.Output.json(code=e.args[0], message=e.args[1]) + else: + return API.Output.json(code=ReturnCode.Server.EXCEPTION, message=repr(e)) + return API.Output.json(code=ReturnCode.Server.EXCEPTION, message=repr(e)) + + @staticmethod + def args_error_response(e): + stat_logger.exception(e) + messages = e.data.get("messages", {}) + return API.Output.json(code=ReturnCode.API.INVALID_PARAMETER, message="Invalid request.", data=messages) + + @staticmethod + def fate_flow_exception(e: FateFlowError): + return API.Output.json(code=e.code, message=e.message) + + @staticmethod + def runtime_exception(code): + def _outer(func): + @wraps(func) + def _wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + if isinstance(e, FateFlowError): + raise e + else: + message = f"Request uri {flask_request.base_url} failed: {str(e)}" + return API.Output.json(code=code, message=message) + return _wrapper + return _outer def get_federated_proxy_address(): @@ -119,3 +144,40 @@ def get_federated_proxy_address(): else: raise RuntimeError(f"can not support coordinate proxy {PROXY_NAME}， all proxy {PROXY.keys()}") return host, port, protocol, PROXY_NAME + + +def generate_headers(party_id, body, initiator_party_id=""): + return HookManager.site_signature( + SignatureParameters(party_id=party_id, body=body, initiator_party_id=initiator_party_id)) + + +def get_exponential_backoff_interval(retries, full_jitter=False): + """Calculate the exponential backoff wait time.""" + # Will be zero if factor equals 0 + countdown = min(REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC * (2 ** retries)) + # Full jitter according to + # https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + if full_jitter: + countdown = random.randrange(countdown + 1) + # Adjust according to maximum wait time and account for negative values. + return max(0, countdown) + + +def federated_coordination_on_http(method, host, port, endpoint, json_body, headers=None, params=None, + timeout=JobDefaultConfig.remote_request_timeout): + url = f'http://{host}:{port}/{API_VERSION}{endpoint}' + for t in range(REQUEST_TRY_TIMES): + try: + response = request( + method=method, url=url, timeout=timeout, + headers=headers, json=json_body, params=params + ) + response.raise_for_status() + except Exception as e: + schedule_logger().warning(f'http api error: {url}\n{e}') + if t >= REQUEST_TRY_TIMES - 1: + raise e + else: + audit_logger().info(f'http api response: {url}\n{response.text}') + return response.json() + time.sleep(get_exponential_backoff_interval(t)) diff --git a/python/fate_flow/utils/base_utils.py b/python/fate_flow/utils/base_utils.py index a3ea7ec14..33b65c4c0 100644 --- a/python/fate_flow/utils/base_utils.py +++ b/python/fate_flow/utils/base_utils.py @@ -13,9 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import base64 import datetime import json import os +import pickle +import random import socket import time import uuid @@ -89,6 +92,18 @@ def bytes_to_string(byte): return byte.decode(encoding="utf-8") +def generate_random_id(length=6, only_number=False): + random_id = '' + if only_number: + chars = '0123456789' + else: + chars = 'AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz0123456789' + len_chars = len(chars) - 1 + for i in range(length): + random_id += chars[random.randint(0, len_chars)] + return random_id + + def json_dumps(src, byte=False, indent=None, with_type=False): dest = json.dumps(src, indent=indent, cls=CustomJSONEncoder, with_type=with_type) if byte: diff --git a/python/fate_flow/utils/conf_utils.py b/python/fate_flow/utils/conf_utils.py index a89350256..318960d9d 100644 --- a/python/fate_flow/utils/conf_utils.py +++ b/python/fate_flow/utils/conf_utils.py @@ -14,11 +14,8 @@ # limitations under the License. # import os -from importlib import import_module -from filelock import FileLock - -from .file_utils import get_project_base_directory, load_yaml_conf, rewrite_yaml_conf, get_fate_flow_directory +from .file_utils import load_yaml_conf, get_fate_flow_directory SERVICE_CONF = "service_conf.yaml" TRANSFER_CONF = "transfer_conf.yaml" @@ -49,39 +46,3 @@ def get_base_config(key, default=None, conf_name=SERVICE_CONF) -> dict: config.update(local_config) return config.get(key, default) if key is not None else config - - -def decrypt_database_password(password): - encrypt_password = get_base_config("encrypt_password", False) - encrypt_module = get_base_config("encrypt_module", False) - private_key = get_base_config("private_key", None) - - if not password or not encrypt_password: - return password - - if not private_key: - raise ValueError("No private key") - - module_fun = encrypt_module.split("#") - pwdecrypt_fun = getattr(import_module(module_fun[0]), module_fun[1]) - - return pwdecrypt_fun(private_key, password) - - -def decrypt_database_config(database=None, passwd_key="passwd"): - if not database: - database = get_base_config("database", {}) - - database[passwd_key] = decrypt_database_password(database[passwd_key]) - return database - - -def update_config(key, value, conf_name=SERVICE_CONF): - conf_path = conf_realpath(conf_name=conf_name) - if not os.path.isabs(conf_path): - conf_path = os.path.join(get_project_base_directory(), conf_path) - - with FileLock(os.path.join(os.path.dirname(conf_path), ".lock")): - config = load_yaml_conf(conf_path=conf_path) or {} - config[key] = value - rewrite_yaml_conf(conf_path=conf_path, config=config) diff --git a/python/fate_flow/utils/data_upload.py b/python/fate_flow/utils/data_upload.py deleted file mode 100644 index e633347e4..000000000 --- a/python/fate_flow/utils/data_upload.py +++ /dev/null @@ -1,259 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import json -import os -import time - -from pydantic import typing - -from fate_flow.engine.storage import Session, EggRollStoreType, StorageEngine, StorageTableMeta, StorageTableOrigin -from fate_flow.entity.engine_types import EngineType -from fate_flow.settings import ENGINES -from fate_flow.utils.file_utils import get_fate_flow_directory -from fate_flow.utils.log import getLogger - -logger = getLogger("upload") - -DEFAULT_ID_DELIMITER = "," -upload_block_max_bytes = 104857600 - - -class Param(object): - def to_dict(self): - d = {} - for k, v in self.__dict__.items(): - if v is None: - continue - d[k] = v - return d - - -class MetaParam(Param): - def __init__(self, - delimiter: str = ",", - label_name: typing.Union[None, str] = None, - label_type: str = "int", - weight_name: typing.Union[None, str] = None, - dtype: str = "float32", - input_format: str = "dense"): - self.delimiter = delimiter - self.label_name = label_name - self.label_type = label_type - self.weight_name = weight_name - self.dtype = dtype - self.input_format = input_format - - -class UploadParam(Param): - def __init__( - self, - file="", - head=1, - partitions=10, - namespace="", - name="", - storage_engine="", - storage_address=None, - destroy=False, - meta=None - ): - self.file = file - self.head = head - self.delimiter = None - self.partitions = partitions - self.namespace = namespace - self.name = name - self.engine = storage_engine - self.storage_address = storage_address - self.destroy = destroy - self.meta = MetaParam(**meta) - - -class Upload: - def __init__(self): - self.MAX_PARTITIONS = 1024 - self.MAX_BYTES = 1024 * 1024 * 8 * 500 - self.parameters: UploadParam = None - self.table = None - self.is_block = False - self.session_id = None - self.session = None - self.schema = {} - - def run(self, parameters: UploadParam): - self.parameters = parameters - self.parameters.delimiter = self.parameters.meta.delimiter - if not self.parameters.engine: - self.parameters.engine = ENGINES.get(EngineType.STORAGE) - logger.info(self.parameters.to_dict()) - storage_engine = parameters.engine - storage_address = parameters.storage_address - if not storage_address: - storage_address = {} - if not os.path.isabs(parameters.file): - parameters.file = os.path.join( - get_fate_flow_directory(), parameters.file - ) - name, namespace = parameters.name, parameters.namespace - read_head = parameters.head - if read_head == 0: - head = False - elif read_head == 1: - head = True - else: - raise Exception("'head' in conf.json should be 0 or 1") - partitions = parameters.partitions - if partitions <= 0 or partitions >= self.MAX_PARTITIONS: - raise Exception( - "Error number of partition, it should between %d and %d" - % (0, self.MAX_PARTITIONS) - ) - with Session() as sess: - if self.parameters.destroy: - table = sess.get_table(namespace=namespace, name=name) - if table: - logger.info( - f"destroy table name: {name} namespace: {namespace} engine: {table.engine}" - ) - try: - table.destroy() - except Exception as e: - logger.error(e) - else: - logger.info( - f"can not found table name: {name} namespace: {namespace}, pass destroy" - ) - address_dict = storage_address.copy() - storage_session = sess.storage( - storage_engine=storage_engine - ) - if storage_engine in {StorageEngine.EGGROLL, StorageEngine.STANDALONE}: - upload_address = { - "name": name, - "namespace": namespace, - "storage_type": EggRollStoreType.ROLLPAIR_LMDB, - } - else: - raise RuntimeError(f"can not support this storage engine: {storage_engine}") - address_dict.update(upload_address) - logger.info(f"upload to {storage_engine} storage, address: {address_dict}") - address = StorageTableMeta.create_address( - storage_engine=storage_engine, address_dict=address_dict - ) - self.table = storage_session.create_table(address=address, origin=StorageTableOrigin.UPLOAD, **self.parameters.to_dict()) - - data_table_count = self.save_data_table(head) - - self.table.meta.update_metas(in_serialized=True) - logger.info("------------load data finish!-----------------") - # rm tmp file - logger.info("file: {}".format(self.parameters.file)) - logger.info("total data_count: {}".format(data_table_count)) - logger.info("table name: {}, table namespace: {}".format(name, namespace)) - return {"name": name, "namespace": namespace, "count": data_table_count} - - def save_data_table(self, head=True): - input_file = self.parameters.file - input_feature_count = self.get_count(input_file) - self.upload_file(input_file, head, input_feature_count) - table_count = self.table.count() - metas_info = { - "count": table_count, - "partitions": self.parameters.partitions - } - if self.parameters.meta: - pass - self.table.meta.update_metas(**metas_info) - return table_count - - def upload_file(self, input_file, head, input_feature_count=None, table=None): - if not table: - table = self.table - with open(input_file, "r") as fin: - lines_count = 0 - if head is True: - data_head = fin.readline() - input_feature_count -= 1 - self.update_table_meta(data_head) - n = 0 - line_index = 0 - while True: - data = list() - lines = fin.readlines(upload_block_max_bytes) - logger.info(upload_block_max_bytes) - if lines: - # self.append_data_line(lines, data, n) - for line in lines: - values = line.rstrip().split(self.parameters.delimiter) - k, v = self.get_data_line( - values=values, - delimiter=self.parameters.delimiter - ) - data.append((k, v)) - line_index += 1 - lines_count += len(data) - table.put_all(data) - else: - return - n += 1 - - def get_count(self, input_file): - with open(input_file, "r", encoding="utf-8") as fp: - count = 0 - for line in fp: - count += 1 - return count - - def generate_table_name(self, input_file_path): - str_time = time.strftime("%Y%m%d%H%M%S", time.localtime()) - file_name = input_file_path.split(".")[0] - file_name = file_name.split("/")[-1] - return file_name, str_time - - def update_table_meta(self, data_head): - logger.info(f"data head: {data_head}") - schema = self.get_header_schema( - header_line=data_head, - delimiter=self.parameters.delimiter - ) - self.schema.update(schema) - self.schema.update(self.parameters.meta.to_dict()) - self.table.put_meta([("schema", self.schema)]) - - def get_header_schema(self, header_line, delimiter): - header_source_item = header_line.split(delimiter) - header = delimiter.join(header_source_item[1:]).strip() - sid = header_source_item[0].strip() - return {'header': header, 'sid': sid} - - def get_data_line(self, values, delimiter, **kwargs): - return values[0], self.list_to_str(values[1:], delimiter=delimiter) - - @staticmethod - def list_to_str(input_list, delimiter): - return delimiter.join(list(map(str, input_list))) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', required=True, type=str, help="runtime conf path") - args = parser.parse_args() - path = args.config - with open(args.config, "r") as f: - conf = json.load(f) - logger.info(conf) - Upload().run(parameters=UploadParam(**conf)) \ No newline at end of file diff --git a/python/fate_flow/utils/engine_utils.py b/python/fate_flow/utils/engine_utils.py index 4d516d388..9b962ea0d 100644 --- a/python/fate_flow/utils/engine_utils.py +++ b/python/fate_flow/utils/engine_utils.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from fate_flow.engine.relation_ship import Relationship -from fate_flow.entity.engine_types import EngineType, FederationEngine, StorageEngine, ComputingEngine -from fate_flow.entity.types import FederatedMode +from fate_flow.entity.types import EngineType, FederationEngine, StorageEngine, ComputingEngine, FederatedMode from fate_flow.utils import conf_utils diff --git a/python/fate_flow/utils/file_utils.py b/python/fate_flow/utils/file_utils.py index 40c4caaaa..2d6c5586b 100644 --- a/python/fate_flow/utils/file_utils.py +++ b/python/fate_flow/utils/file_utils.py @@ -17,97 +17,66 @@ import json import os -from cachetools import LRUCache, cached from ruamel import yaml -PROJECT_BASE = os.getenv("FATE_PROJECT_BASE") or os.getenv("FATE_DEPLOY_BASE") -FATE_BASE = os.getenv("FATE_BASE") -READTHEDOC = os.getenv("READTHEDOC") +from fate_flow.runtime.env import is_in_virtualenv + +PROJECT_BASE = os.getenv("FATE_PROJECT_BASE") +FATE_PYTHON_PATH = os.getenv("FATE_PYTHONPATH") def get_project_base_directory(*args): global PROJECT_BASE - global READTHEDOC if PROJECT_BASE is None: PROJECT_BASE = os.path.abspath( os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, + os.pardir, ) ) - if READTHEDOC is None: - PROJECT_BASE = os.path.abspath( - os.path.join( - PROJECT_BASE, - os.pardir, - ) - ) if args: return os.path.join(PROJECT_BASE, *args) return PROJECT_BASE -def get_fate_flow_directory(*args): - FATE_FLOW_BASE = os.path.abspath( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - os.pardir, - os.pardir, - os.pardir, - ) - ) - if args: - return os.path.join(FATE_FLOW_BASE, *args) - return FATE_FLOW_BASE +def get_fate_python_path(): + global FATE_PYTHON_PATH + if FATE_PYTHON_PATH is None: + FATE_PYTHON_PATH = get_project_base_directory("fate", "python") + if not os.path.exists(FATE_PYTHON_PATH): + FATE_PYTHON_PATH = get_project_base_directory("python") + if not os.path.exists(FATE_PYTHON_PATH): + return + return FATE_PYTHON_PATH -@cached(cache=LRUCache(maxsize=10)) -def load_json_conf(conf_path): - if os.path.isabs(conf_path): - json_conf_path = conf_path - else: - json_conf_path = os.path.join(get_project_base_directory(), conf_path) - try: - with open(json_conf_path) as f: - return json.load(f) - except BaseException: - raise EnvironmentError( - "loading json file config from '{}' failed!".format(json_conf_path) +def get_fate_flow_directory(*args): + if is_in_virtualenv(): + fate_flow_dir = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + os.pardir + ) ) - - -def dump_json_conf(config_data, conf_path): - if os.path.isabs(conf_path): - json_conf_path = conf_path else: - json_conf_path = os.path.join(get_project_base_directory(), conf_path) - try: - with open(json_conf_path, "w") as f: - json.dump(config_data, f, indent=4) - except BaseException: - raise EnvironmentError( - "loading json file config from '{}' failed!".format(json_conf_path) - ) - - -def load_json_conf_real_time(conf_path): - if os.path.isabs(conf_path): - json_conf_path = conf_path - else: - json_conf_path = os.path.join(get_project_base_directory(), conf_path) - try: - with open(json_conf_path) as f: - return json.load(f) - except BaseException: - raise EnvironmentError( - "loading json file config from '{}' failed!".format(json_conf_path) + fate_flow_dir = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + os.pardir, + os.pardir, + os.pardir, + ) ) + if args: + return os.path.join(fate_flow_dir, *args) + return fate_flow_dir def load_yaml_conf(conf_path): if not os.path.isabs(conf_path): - conf_path = os.path.join(get_project_base_directory(), conf_path) + conf_path = os.path.join(get_fate_flow_directory(), conf_path) try: with open(conf_path) as f: return yaml.safe_load(f) @@ -117,18 +86,6 @@ def load_yaml_conf(conf_path): ) -def rewrite_yaml_conf(conf_path, config): - if not os.path.isabs(conf_path): - conf_path = os.path.join(get_project_base_directory(), conf_path) - try: - with open(conf_path, "w") as f: - yaml.dump(config, f, Dumper=yaml.RoundTripDumper) - except Exception as e: - raise EnvironmentError( - "rewrite yaml file config {} failed:".format(conf_path), e - ) - - def rewrite_json_file(filepath, json_data): with open(filepath, "w") as f: json.dump(json_data, f, indent=4, separators=(",", ": ")) diff --git a/python/fate_flow/utils/grpc_utils.py b/python/fate_flow/utils/grpc_utils.py index d2e291cd8..9207e2ff2 100644 --- a/python/fate_flow/utils/grpc_utils.py +++ b/python/fate_flow/utils/grpc_utils.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import grpc - +from fate_flow.errors.server_error import ResponseException from fate_flow.proto.osx import osx_pb2, osx_pb2_grpc from fate_flow.proto.rollsite import proxy_pb2_grpc, basic_meta_pb2, proxy_pb2 from fate_flow.runtime.runtime_config import RuntimeConfig -from fate_flow.settings import FATE_FLOW_SERVICE_NAME, GRPC_PORT, HOST, REMOTE_REQUEST_TIMEOUT +from fate_flow.runtime.system_settings import FATE_FLOW_SERVICE_NAME, GRPC_PORT, HOST, REMOTE_REQUEST_TIMEOUT from fate_flow.utils.base_utils import json_loads, json_dumps from fate_flow.utils.log_utils import audit_logger from fate_flow.utils.requests_utils import request @@ -78,8 +77,8 @@ def unaryCall(self, _request, context): audit_logger(job_id).info('rpc receive: {}'.format(packet)) audit_logger(job_id).info("rpc receive: {} {}".format(get_url(_suffix), param)) resp = request(method=method, url=get_url(_suffix), json=param_dict, headers=headers) - resp_json = resp.json() - + audit_logger(job_id).info(f"resp: {resp.text}") + resp_json = response_json(resp) return wrap_grpc_packet(resp_json, method, _suffix, dst.partyId, src.partyId, job_id) @@ -100,7 +99,7 @@ def invoke(self, _request, context): resp = request(method=request_info.get("method"), url=get_url(request_info.get("uri")), json=request_info.get("json_body"), headers=request_info.get("headers", {})) audit_logger(job_id).info(f"resp: {resp.text}") - resp_json = resp.json() + resp_json = response_json(resp) _meta = { "TechProviderCode": metadata.get("TechProviderCode", ""), "SourceInstID": metadata.get("TargetInstID", ""), @@ -115,3 +114,13 @@ def invoke(self, _request, context): res = osx_pb2.Outbound(metadata=_meta, payload=_data) audit_logger(job_id).info(f"response: {res}") return res + + +def response_json(response): + try: + return response.json() + except: + audit_logger().exception(response.text) + e = ResponseException(response=response.text) + return {"code": e.code, "message": e.message} + diff --git a/python/fate_flow/utils/io_utils.py b/python/fate_flow/utils/io_utils.py new file mode 100644 index 000000000..9f0f3bf2e --- /dev/null +++ b/python/fate_flow/utils/io_utils.py @@ -0,0 +1,208 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import re +from abc import ABCMeta +from dataclasses import dataclass +from typing import Optional + +# see https://www.rfc-editor.org/rfc/rfc3986#appendix-B +# scheme = $2 +# authority = $4 +# path = $5 +# query = $7 +# fragment = $9 +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME + +_uri_regex = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + + +@dataclass +class URI: + schema: str + path: str + query: Optional[str] = None + fragment: Optional[str] = None + authority: Optional[str] = None + + @classmethod + def from_string(cls, uri: str) -> "URI": + match = _uri_regex.fullmatch(uri) + if match is None: + raise ValueError(f"`{uri}` is not valid uri") + _, schema, _, authority, path, _, query, _, fragment = match.groups() + return URI(schema, path, query, fragment, authority) + + def to_schema(self): + for cls in ConcrateURI.__subclasses__(): + if cls.schema() == self.schema: + return cls.from_uri(self) + raise NotImplementedError(f"uri schema `{self.schema}` not found") + + +class ConcrateURI(metaclass=ABCMeta): + @classmethod + def schema(cls) -> str: + ... + + @classmethod + def from_uri(cls, uri: URI) -> "ConcrateURI": + ... + + def create_file(self, name): + ... + + def to_string(self): + ... + + +_EGGROLL_NAME_MAX_SIZE = 128 + + +@dataclass +class FileURI(ConcrateURI): + path: str + + @classmethod + def schema(cls): + return "file" + + @classmethod + def from_uri(cls, uri: URI): + return FileURI(uri.path) + + def create_file(self, name): + return FileURI(f"{self.path}/{name}") + + def to_string(self): + return f"file://{self.path}" + + +@dataclass +class HttpURI(ConcrateURI): + path: str + + @classmethod + def schema(cls): + return "http" + + @classmethod + def from_uri(cls, uri: URI): + return HttpURI(uri.path) + + def create_file(self, name): + return HttpURI(path=f"{self.path}/{name}") + + def to_string(self): + return f"{self.path}" + + +@dataclass +class EggrollURI(ConcrateURI): + namespace: str + name: str + + @classmethod + def schema(cls): + return "eggroll" + + @classmethod + def from_uri(cls, uri: URI): + _, namespace, *names = uri.path.split("/") + name = "_".join(names) + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return EggrollURI(namespace, name) + + def create_file(self, name): + name = f"{self.name}_{name}" + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return EggrollURI(namespace=self.namespace, name=name) + + def to_string(self): + return f"eggroll:///{self.namespace}/{self.name}" + + +@dataclass +class StandaloneURI(ConcrateURI): + namespace: str + name: str + + @classmethod + def schema(cls): + return "standalone" + + @classmethod + def from_uri(cls, uri: URI): + if STANDALONE_DATA_HOME in uri.path: + _, namespace, *names = uri.path.split(STANDALONE_DATA_HOME)[1].split("/") + else: + _, namespace, *names = uri.path.split("/") + name = "_".join(names) + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return StandaloneURI(namespace, name) + + def create_file(self, name): + name = f"{self.name}_{name}" + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return StandaloneURI(namespace=self.namespace, name=name) + + def to_string(self): + return f"standalone:///{self.namespace}/{self.name}" + + +@dataclass +class HdfsURI(ConcrateURI): + path: str + authority: Optional[str] = None + + @classmethod + def schema(cls): + return "hdfs" + + @classmethod + def from_uri(cls, uri: URI): + return HdfsURI(uri.path, uri.authority) + + def create_file(self, name): + return HdfsURI(path=f"{self.path}/{name}", authority=self.authority) + + def to_string(self): + if self.authority: + return f"hdfs://{self.authority}{self.path}" + else: + return f"hdfs://{self.path}" + + +@dataclass +class LocalfsURI(ConcrateURI): + path: str + + @classmethod + def schema(cls): + return "path" + + @classmethod + def from_uri(cls, uri: URI): + return LocalfsURI(uri.path) + + def create_file(self, name): + return LocalfsURI(path=f"{self.path}/{name}") + + def to_string(self): + return f"{self.path}" diff --git a/python/fate_flow/utils/job_utils.py b/python/fate_flow/utils/job_utils.py index d691fe006..692239ea3 100644 --- a/python/fate_flow/utils/job_utils.py +++ b/python/fate_flow/utils/job_utils.py @@ -16,11 +16,13 @@ import os import threading +import yaml + from fate_flow.db.base_models import DB -from fate_flow.db.db_models import Job -from fate_flow.entity.dag_structures import DAGSchema +from fate_flow.db.db_models import Job, Task +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.runtime.system_settings import LOG_DIR, JOB_DIR, WORKERS_DIR from fate_flow.utils.base_utils import fate_uuid -from fate_flow.utils.file_utils import get_fate_flow_directory class JobIdGenerator(object): @@ -35,7 +37,6 @@ def next_id(self): """ generate next job id with locking """ - #todo: there is duplication in the case of multiple instances deployment now = datetime.datetime.now() with JobIdGenerator._lock: if self._pre_timestamp == now: @@ -76,44 +77,67 @@ def generate_session_id(task_id, task_version, role, party_id, suffix=None, rand def get_job_directory(job_id, *args): - return os.path.join(get_fate_flow_directory(), 'jobs', job_id, *args) + return os.path.join(JOB_DIR, job_id, *args) def get_job_log_directory(job_id, *args): - return os.path.join(get_fate_flow_directory(), 'logs', job_id, *args) - - -def get_task_directory(job_id, role, party_id, task_name, task_id, task_version, **kwargs): - return get_job_directory(job_id, role, party_id, task_name, task_id, str(task_version)) + return os.path.join(LOG_DIR, job_id, *args) -def start_session_stop(task): - # todo: session stop - pass +def get_task_directory(job_id, role, party_id, task_name, task_version, input=False, output=False, **kwargs): + if input: + return get_job_directory(job_id, role, party_id, task_name, str(task_version), "input") + if output: + return get_job_directory(job_id, role, party_id, task_name, str(task_version), "output") + else: + return get_job_directory(job_id, role, party_id, task_name, str(task_version)) def get_general_worker_directory(worker_name, worker_id, *args): - return os.path.join(get_fate_flow_directory(), worker_name, worker_id, *args) + return os.path.join(WORKERS_DIR, worker_name, worker_id, *args) def get_general_worker_log_directory(worker_name, worker_id, *args): - return os.path.join(get_fate_flow_directory(), 'logs', worker_name, worker_id, *args) + return os.path.join(LOG_DIR, worker_name, worker_id, *args) def generate_model_info(job_id): model_id = job_id - model_version = 0 + model_version = "0" return model_id, model_version @DB.connection_context() def get_job_resource_info(job_id, role, party_id): - jobs = Job.select(Job.f_dag).where(Job.f_job_id == job_id, - Job.f_role == role, - Job.f_party_id == party_id) + jobs = Job.select(Job.f_cores, Job.f_memory).where( + Job.f_job_id == job_id, + Job.f_role == role, + Job.f_party_id == party_id) if jobs: job = jobs[0] - dag_schema = DAGSchema(**job.f_dag) - return dag_schema.dag.conf.task_cores, dag_schema.dag.conf.task_parallelism + return job.f_cores, job.f_memory else: return None, None + + +@DB.connection_context() +def get_task_resource_info(job_id, role, party_id, task_id, task_version): + tasks = Task.select(Task.f_task_cores, Task.f_memory).where( + Task.f_job_id == job_id, + Task.f_role == role, + Task.f_party_id == party_id, + Task.f_task_id == task_id, + Task.f_task_version == task_version + ) + if tasks: + task = tasks[0] + return task.f_task_cores, task.f_memory + else: + return None, None + + +def save_job_dag(job_id, dag): + job_conf_file = os.path.join(JOB_DIR, job_id, "dag.yaml") + os.makedirs(os.path.dirname(job_conf_file), exist_ok=True) + with open(job_conf_file, "w") as f: + f.write(yaml.dump(dag)) diff --git a/python/fate_flow/utils/log.py b/python/fate_flow/utils/log.py index 842ed8305..e8abfe320 100644 --- a/python/fate_flow/utils/log.py +++ b/python/fate_flow/utils/log.py @@ -21,7 +21,7 @@ from logging.handlers import TimedRotatingFileHandler from threading import RLock -from fate_flow.utils.file_utils import get_fate_flow_directory +from fate_flow.runtime.system_settings import LOG_SHARE class LoggerFactory(object): @@ -33,7 +33,7 @@ class LoggerFactory(object): LOG_DIR = None PARENT_LOG_DIR = None - log_share = True + log_share = LOG_SHARE append_to_parent_log = None @@ -50,14 +50,12 @@ class LoggerFactory(object): schedule_logger_dict = {} @staticmethod - def set_directory(directory=None, parent_log_dir=None, append_to_parent_log=None, force=False): + def set_directory(directory, parent_log_dir=None, append_to_parent_log=None, force=False): if parent_log_dir: LoggerFactory.PARENT_LOG_DIR = parent_log_dir if append_to_parent_log: LoggerFactory.append_to_parent_log = append_to_parent_log with LoggerFactory.lock: - if not directory: - directory = get_fate_flow_directory() if not LoggerFactory.LOG_DIR or force: LoggerFactory.LOG_DIR = directory if LoggerFactory.log_share: @@ -172,6 +170,8 @@ def init_logger(class_name): @staticmethod def assemble_global_handler(logger): + if isinstance(LoggerFactory.LEVEL, str): + LoggerFactory.LEVEL = logging._nameToLevel[LoggerFactory.LEVEL] if LoggerFactory.LOG_DIR: for level in LoggerFactory.levels: if level >= LoggerFactory.LEVEL: diff --git a/python/fate_flow/utils/log_utils.py b/python/fate_flow/utils/log_utils.py index 4579d31f4..0e119489d 100644 --- a/python/fate_flow/utils/log_utils.py +++ b/python/fate_flow/utils/log_utils.py @@ -19,7 +19,7 @@ import traceback import logging -from fate_flow.utils.file_utils import get_fate_flow_directory +from fate_flow.runtime.system_settings import FATE_FLOW_LOG_DIR, LOG_DIR from fate_flow.utils.log import LoggerFactory, getLogger @@ -67,14 +67,9 @@ def exception_to_trace_string(ex): return "".join(traceback.TracebackException.from_exception(ex).format()) -def get_logger_base_dir(): - job_log_dir = get_fate_flow_directory('logs') - return job_log_dir - - def get_job_logger(job_id, log_type): - fate_flow_log_dir = get_fate_flow_directory('logs', 'fate_flow') - job_log_dir = get_fate_flow_directory('logs', job_id) + fate_flow_log_dir = FATE_FLOW_LOG_DIR + job_log_dir = os.path.join(LOG_DIR, job_id) if not job_id: log_dirs = [fate_flow_log_dir] else: diff --git a/python/fate_flow/utils/password_utils.py b/python/fate_flow/utils/password_utils.py new file mode 100644 index 000000000..6db4ff75e --- /dev/null +++ b/python/fate_flow/utils/password_utils.py @@ -0,0 +1,43 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os.path +from importlib import import_module + +from fate_flow.runtime.system_settings import ENCRYPT_CONF +from fate_flow.utils.conf_utils import conf_realpath + + +def decrypt_database_config(database, passwd_key="passwd", decrypt_key=""): + database[passwd_key] = decrypt_password(database[passwd_key], key=decrypt_key) + return database + + +def decrypt_password(password, key=""): + if not ENCRYPT_CONF or not key or key not in ENCRYPT_CONF: + return password + encrypt_module = ENCRYPT_CONF.get(key).get("module", "") + private_path = ENCRYPT_CONF.get(key).get("private_path", "") + if not encrypt_module: + raise ValueError(f"module is {encrypt_module}") + if not private_path: + raise ValueError(f"private_path is {private_path}") + if not os.path.isabs(private_path): + private_path = conf_realpath(private_path) + with open(private_path) as f: + private_key = f.read() + module_func = encrypt_module.split("#") + encrypt_func = getattr(import_module(module_func[0]), module_func[1]) + return encrypt_func(private_key, password) diff --git a/python/fate_flow/utils/permission_utils.py b/python/fate_flow/utils/permission_utils.py new file mode 100644 index 000000000..92ebc557d --- /dev/null +++ b/python/fate_flow/utils/permission_utils.py @@ -0,0 +1,34 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.hook.common.parameters import PermissionCheckParameters +from fate_flow.hub.flow_hub import FlowHub + + +def get_permission_parameters(role, party_id, initiator_party_id, job_info) -> PermissionCheckParameters: + dag_schema = DAGSchema(**job_info['dag_schema']) + job_parser = FlowHub.load_job_parser(dag_schema) + component_list = job_parser.component_ref_list(role, party_id) + dataset_list = job_parser.dataset_list(role, party_id) + component_parameters = job_parser.role_parameters(role, party_id) + return PermissionCheckParameters( + initiator_party_id=initiator_party_id, + roles=dag_schema.dag.parties, + component_list=component_list, + dataset_list=dataset_list, + dag_schema=dag_schema.dict(), + component_parameters=component_parameters + ) diff --git a/python/fate_flow/utils/process_utils.py b/python/fate_flow/utils/process_utils.py index fa03b1268..8fa085290 100644 --- a/python/fate_flow/utils/process_utils.py +++ b/python/fate_flow/utils/process_utils.py @@ -14,31 +14,40 @@ # limitations under the License. # import errno +import json import os import subprocess import time import psutil + +from fate_flow.entity.code import KillProcessRetCode +from fate_flow.utils.log import getLogger from fate_flow.utils.log_utils import schedule_logger from fate_flow.db.db_models import Task -from fate_flow.entity.types import KillProcessRetCode, ProcessRole -from fate_flow.settings import SUBPROCESS_STD_LOG_NAME -from fate_flow.settings import stat_logger +from fate_flow.entity.types import ProcessRole + +stat_logger = getLogger() -def run_subprocess(job_id, config_dir, process_cmd, added_env: dict = None, log_dir=None, cwd_dir=None, process_name="", process_id=""): +def run_subprocess( + job_id, config_dir, process_cmd, process_name, added_env: dict = None, std_dir=None, cwd_dir=None, stderr=None +): logger = schedule_logger(job_id) if job_id else stat_logger process_cmd = [str(cmd) for cmd in process_cmd] logger.info("start process command: \n{}".format(" ".join(process_cmd))) os.makedirs(config_dir, exist_ok=True) - if not log_dir: - log_dir = config_dir - if log_dir: - os.makedirs(log_dir, exist_ok=True) - std_path = get_std_path(log_dir=log_dir, process_name=process_name, process_id=process_id) + os.makedirs(std_dir, exist_ok=True) + if not std_dir: + std_dir = config_dir + std_path = get_std_path(std_dir=std_dir, process_name=process_name) + std = open(std_path, 'w') - pid_path = os.path.join(config_dir, f"{process_name}_pid") + if not stderr: + stderr = std + pid_path = os.path.join(config_dir, "pid", f"{process_name}") + os.makedirs(os.path.dirname(pid_path), exist_ok=True) if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() @@ -51,14 +60,15 @@ def run_subprocess(job_id, config_dir, process_cmd, added_env: dict = None, log_ subprocess_env["PROCESS_ROLE"] = ProcessRole.WORKER.value if added_env: for name, value in added_env.items(): + if not value: + continue if name.endswith("PATH") and subprocess_env.get(name) is not None: value += ':' + subprocess_env[name] subprocess_env[name] = value - subprocess_env.pop("CLASSPATH", None) - + logger.info(f"RUN ENV：{json.dumps(subprocess_env)}") p = subprocess.Popen(process_cmd, stdout=std, - stderr=std, + stderr=stderr, startupinfo=startupinfo, cwd=cwd_dir, env=subprocess_env @@ -126,15 +136,8 @@ def check_process_by_cmdline(actual: list, expected: list): return True -def get_std_path(log_dir, process_name="", process_id=""): - std_log_path = f"{process_name}_{process_id}_{SUBPROCESS_STD_LOG_NAME}" if process_name else SUBPROCESS_STD_LOG_NAME - return os.path.join(log_dir, std_log_path) - - -def get_subprocess_std(log_dir, process_name="", process_id=""): - with open(get_std_path(log_dir, process_name, process_id), "r") as fr: - text = fr.read() - return text +def get_std_path(std_dir, process_name): + return os.path.join(std_dir, process_name) def wait_child_process(signum, frame): @@ -204,10 +207,10 @@ def kill_task_executor_process(task: Task, only_child=False): return KillProcessRetCode.ERROR_PID for child in p.children(recursive=True): if check_process(pid=child.pid, task=task): - kill(p) + child.kill() if not only_child: if check_process(pid, task=task): - kill(p) + p.kill() schedule_logger(task.f_job_id).info("successfully stop task {} {} {} process pid:{}".format( task.f_task_id, task.f_role, task.f_party_id, pid)) return KillProcessRetCode.KILLED diff --git a/python/fate_flow/utils/schedule_utils.py b/python/fate_flow/utils/schedule_utils.py index e8e27e17d..5c82190c1 100644 --- a/python/fate_flow/utils/schedule_utils.py +++ b/python/fate_flow/utils/schedule_utils.py @@ -16,6 +16,7 @@ from fate_flow.db.base_models import DB from fate_flow.db.schedule_models import ScheduleJob from fate_flow.utils.base_utils import current_timestamp +from fate_flow.utils.log_utils import schedule_logger @DB.connection_context() @@ -33,4 +34,29 @@ def rerun_signal(job_id, set_or_reset: bool): else: raise RuntimeError(f"can not support rereun signal {set_or_reset}") update_status = ScheduleJob.update(update_fields).where(ScheduleJob.f_job_id == job_id).execute() > 0 - return update_status \ No newline at end of file + return update_status + + +@DB.connection_context() +def schedule_signal(job_id: object, set_or_reset: bool) -> bool: + filters = [ScheduleJob.f_job_id == job_id] + if set_or_reset: + update_fields = {ScheduleJob.f_schedule_signal: True, ScheduleJob.f_schedule_time: current_timestamp()} + filters.append(ScheduleJob.f_schedule_signal == False) + else: + update_fields = {ScheduleJob.f_schedule_signal: False, ScheduleJob.f_schedule_time: None} + filters.append(ScheduleJob.f_schedule_signal == True) + update_status = ScheduleJob.update(update_fields).where(*filters).execute() > 0 + if set_or_reset and not update_status: + # update timeout signal + schedule_timeout_signal(job_id) + return update_status + + +def schedule_timeout_signal(job_id, ready_timeout_ttl: int = 10*6000): + job_list = ScheduleJob.query(job_id=job_id, schedule_signal=True) + if job_list: + job = job_list[0] + if current_timestamp() - job.f_schedule_time > ready_timeout_ttl: + schedule_logger(job_id).info("schedule timeout, try to update signal") + schedule_signal(job_id, set_or_reset=False) diff --git a/python/fate_flow/utils/version.py b/python/fate_flow/utils/version.py index 87eeff58f..9812830a9 100644 --- a/python/fate_flow/utils/version.py +++ b/python/fate_flow/utils/version.py @@ -18,14 +18,18 @@ import dotenv import typing -from fate_flow.utils.file_utils import get_project_base_directory +from fate_flow.runtime.system_settings import VERSION_FILE_PATH def get_versions() -> typing.Mapping[str, typing.Any]: return dotenv.dotenv_values( - dotenv_path=os.path.join(get_project_base_directory(), "fateflow.env") + dotenv_path=VERSION_FILE_PATH ) def get_flow_version() -> typing.Optional[str]: - return get_versions().get("FATEFlow") \ No newline at end of file + return get_versions().get("FATEFlow") + + +def get_default_fate_version() -> typing.Optional[str]: + return get_versions().get("FATE") diff --git a/python/fate_flow/utils/wraps_utils.py b/python/fate_flow/utils/wraps_utils.py new file mode 100644 index 000000000..08848800b --- /dev/null +++ b/python/fate_flow/utils/wraps_utils.py @@ -0,0 +1,247 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import threading +from functools import wraps + +from fate_flow.entity.code import ReturnCode + +from flask import request as flask_request +from fate_flow.errors.server_error import NoFoundTask, ResponseException, NoFoundINSTANCE, NoPermission +from fate_flow.hook import HookManager +from fate_flow.operation.job_saver import JobSaver +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import HOST, HTTP_PORT, API_VERSION, PERMISSION_SWITCH +from fate_flow.utils.api_utils import API, federated_coordination_on_http +from fate_flow.utils.log_utils import schedule_logger +from fate_flow.utils.permission_utils import get_permission_parameters +from fate_flow.utils.requests_utils import request +from fate_flow.utils.schedule_utils import schedule_signal +from fate_flow.db.casbin_models import FATE_CASBIN + + +def filter_parameters(filter_value=None): + def _inner(func): + @wraps(func) + def _wrapper(*args, **kwargs): + _kwargs = {} + for k, v in kwargs.items(): + if v != filter_value: + _kwargs[k] = v + return func(*args, **_kwargs) + return _wrapper + return _inner + + +def switch_function(switch, code=ReturnCode.Server.FUNCTION_RESTRICTED, message="function restricted"): + def _inner(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if switch: + return func(*args, **kwargs) + else: + raise Exception(code, f"func {func.__name__}, {message}") + return _wrapper + return _inner + + +def task_request_proxy(filter_local=False, force=True): + def _outer(func): + @wraps(func) + def _wrapper(*args, **kwargs): + party_id, role, task_id, task_version = kwargs.get("party_id"), kwargs.get("role"), \ + kwargs.get("task_id"), kwargs.get("task_version") + if not filter_local or (filter_local and role == "local"): + tasks = JobSaver.query_task(task_id=task_id, task_version=task_version, role=role, party_id=party_id) + if tasks: + if tasks[0].f_run_ip and tasks[0].f_run_port: + if tasks[0].f_run_ip != RuntimeConfig.JOB_SERVER_HOST: + source_url = flask_request.url + source_address = source_url.split("/")[2] + dest_address = ":".join([tasks[0].f_run_ip, str(tasks[0].f_run_port)]) + dest_url = source_url.replace(source_address, dest_address) + try: + response = request(method=flask_request.method, url=dest_url, json=flask_request.json, + headers=flask_request.headers, params=flask_request.args) + if 200 <= response.status_code < 300: + response = response.json() + return API.Output.json(code=response.get("code"), message=response.get("message")) + else: + raise ResponseException(response=response.text) + except Exception as e: + if force: + return func(*args, **kwargs) + raise e + else: + return API.Output.fate_flow_exception(NoFoundTask( + role=role, + party_id=party_id, + task_id=task_id, + task_version=task_version + )) + return func(*args, **kwargs) + return _wrapper + return _outer + + +def cluster_route(func): + @wraps(func) + def _route(*args, **kwargs): + instance_id = kwargs.get('instance_id') + request_data = flask_request.json or flask_request.form.to_dict() + if not instance_id: + return func(*args, **kwargs) + instance = RuntimeConfig.SERVICE_DB.get_servers().get(instance_id) + if instance is None: + return API.Output.fate_flow_exception(NoFoundINSTANCE(instance_id=instance_id)) + + if instance.http_address == f'{HOST}:{HTTP_PORT}': + return func(*args, **kwargs) + + endpoint = flask_request.full_path + prefix = f'/{API_VERSION}/' + if endpoint.startswith(prefix): + endpoint = endpoint[len(prefix) - 1:] + response = federated_coordination_on_http( + method=flask_request.method, + host=instance.host, + port=instance.http_port, + endpoint=endpoint, + json_body=request_data, + headers=flask_request.headers, + ) + return API.Output.json(**response) + return _route + + +def schedule_lock(func): + @wraps(func) + def _wrapper(*args, **kwargs): + _lock = kwargs.pop("lock", False) + if _lock: + job = kwargs.get("job") + schedule_logger(job.f_job_id).debug(f"get job {job.f_job_id} schedule lock") + _result = None + if not schedule_signal(job_id=job.f_job_id, set_or_reset=True): + schedule_logger(job.f_job_id).warn(f"get job {job.f_job_id} schedule lock failed, " + f"job may be handled by another scheduler") + return + try: + _result = func(*args, **kwargs) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + raise e + finally: + schedule_signal(job_id=job.f_job_id, set_or_reset=False) + schedule_logger(job.f_job_id).debug(f"release job {job.f_job_id} schedule lock") + return _result + else: + return func(*args, **kwargs) + return _wrapper + + +def threading_lock(func): + @wraps(func) + def _wrapper(*args, **kwargs): + with threading.Lock(): + return func(*args, **kwargs) + return _wrapper + + +def create_job_request_check(func): + @wraps(func) + def _wrapper(*_args, **_kwargs): + party_id = _kwargs.get("party_id") + role = _kwargs.get("role") + body = flask_request.json + headers = flask_request.headers + initiator_party_id = headers.get("initiator_party_id") + + # permission check + if PERMISSION_SWITCH: + permission_return = HookManager.permission_check(get_permission_parameters( + role, party_id, initiator_party_id, body + )) + if permission_return.code != ReturnCode.Base.SUCCESS: + return API.Output.fate_flow_exception(permission_return) + return func(*_args, **_kwargs) + return _wrapper + + +def check_permission(operate=None, types=None): + def _inner(func): + @wraps(func) + def _wrapper(*args, **kwargs): + _init = kwargs.get("init", False) + if not _init: + conf_app_id = flask_request.headers.get("Appid") + conf_roles_dct = [roles for roles in FATE_CASBIN.get_roles_for_user(conf_app_id)] + if conf_app_id == "admin": + conf_role = conf_app_id + elif len(conf_roles_dct): + if "super_client" in conf_roles_dct: + conf_role = "super_client" + else: + conf_role = "client" + else: + raise NoPermission + if types == "client": + app_id = kwargs.get("app_id") + if app_id != "admin": + app_id_role = "super_client" if FATE_CASBIN.has_role_for_user(app_id, "super_client") else "client" + else: + app_id_role = "admin" + if operate == "query": + if conf_role == "super_client": + if conf_app_id != app_id: + if app_id_role != "client": + raise NoPermission + if conf_role == "client" and conf_app_id != app_id: + raise NoPermission + if operate == "delete" and ( + app_id == conf_app_id + or (conf_role == "super_client" and app_id_role in ["admin", "super_client"]) + or conf_role == "client"): + raise NoPermission + if operate == "create" and conf_role == "client": raise NoPermission + + if types == "permission": + app_id = kwargs.get("app_id") + if app_id != "admin": + app_id_role = FATE_CASBIN.has_role_for_user(app_id, "super_client") + app_id_role = "super_client" if app_id_role else "client" + else: + app_id_role = "admin" + if operate == "query" and conf_role == "client" and conf_app_id != app_id: + raise NoPermission + if operate == "grant": + grant_role = kwargs.get("grant_role", False) + if not grant_role: + grant_role = kwargs.get("role", False) + if conf_role == "admin" and app_id_role == "admin": raise NoPermission + if conf_role == "super_client": + app_id_role_client = FATE_CASBIN.get_roles_for_user(app_id) + if app_id_role_client: raise NoPermission + if grant_role == "super_client": raise NoPermission + if conf_role == "client": raise NoPermission + if operate == "delete": + grant_role = kwargs.get("grant_role", None) + if grant_role and conf_role == "super_client" and grant_role == "super_client":raise NoPermission + if conf_role == app_id_role: raise NoPermission + if conf_role == "super_client" and app_id_role != "client":raise NoPermission + if conf_role == "client":raise NoPermission + return func(*args, **kwargs) + return _wrapper + return _inner diff --git a/python/fate_flow/utils/xthread.py b/python/fate_flow/utils/xthread.py index d2973b6d3..e17c56c58 100644 --- a/python/fate_flow/utils/xthread.py +++ b/python/fate_flow/utils/xthread.py @@ -26,7 +26,10 @@ import threading import weakref import os -from fate_flow.settings import stat_logger + +from fate_flow.utils.log import getLogger + +stat_logger = getLogger() # Workers are created as daemon threads. This is done to allow the interpreter # to exit when there are still idle threads in a ThreadPoolExecutor's thread @@ -54,8 +57,10 @@ def _python_exit(): for t, q in items: t.join() + atexit.register(_python_exit) + class _WorkItem(object): def __init__(self, future, fn, args, kwargs): self.future = future @@ -76,6 +81,7 @@ def run(self): else: self.future.set_result(result) + def _worker(executor_reference, work_queue): try: while True: @@ -99,8 +105,8 @@ def _worker(executor_reference, work_queue): except BaseException: _base.LOGGER.critical('Exception in worker', exc_info=True) -class ThreadPoolExecutor(_base.Executor): +class ThreadPoolExecutor(_base.Executor): # Used to assign unique thread names when thread_name_prefix is not supplied. _counter = itertools.count().__next__ @@ -127,6 +133,10 @@ def __init__(self, max_workers=None, thread_name_prefix=''): self._thread_name_prefix = (thread_name_prefix or ("ThreadPoolExecutor-%d" % self._counter())) + @property + def max_workers(self): + return self._max_workers + def submit(self, fn, *args, **kwargs): with self._shutdown_lock: if self._shutdown: diff --git a/python/fate_flow/worker/fate_executor.py b/python/fate_flow/worker/fate_executor.py new file mode 100644 index 000000000..da1e79481 --- /dev/null +++ b/python/fate_flow/worker/fate_executor.py @@ -0,0 +1,26 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class FateSubmit: + @staticmethod + def run(): + import runpy + runpy.run_module(mod_name='fate.components', run_name='__main__') + + +if __name__ == "__main__": + FateSubmit.run() diff --git a/python/fate_flow/worker/fate_flow_executor.py b/python/fate_flow/worker/fate_flow_executor.py new file mode 100644 index 000000000..a25ad49fb --- /dev/null +++ b/python/fate_flow/worker/fate_flow_executor.py @@ -0,0 +1,35 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import sys + +from fate_flow.entity import BaseEntity +from fate_flow.utils.log import getLogger + + +class FateFlowSubmit: + @staticmethod + def run(): + import click + from fate_flow.entrypoint.cli import component + + cli = click.Group() + cli.add_command(component) + cli(prog_name="python -m fate_flow.components") + + +if __name__ == "__main__": + FateFlowSubmit.run() diff --git a/python/ofx/api/client.py b/python/ofx/api/client.py index 150ae3de6..bcc47d774 100644 --- a/python/ofx/api/client.py +++ b/python/ofx/api/client.py @@ -23,7 +23,8 @@ class FlowSchedulerApi: A client for communicating with a flow server. """ def __init__(self, host="127.0.0.1", port=9380, protocol="http", api_version=None, timeout=60, - remote_protocol="http", remote_host=None, remote_port=None, grpc_channel="default"): + remote_protocol="http", remote_host=None, remote_port=None, grpc_channel="default", + callback=None): self.client = APIClient( host=host, port=port, @@ -35,6 +36,6 @@ def __init__(self, host="127.0.0.1", port=9380, protocol="http", api_version=Non remote_protocol=remote_protocol, grpc_channel=grpc_channel ) - self.federated = Federated(client=self.client) - self.scheduler = Scheduler(client=self.client) - self.worker = Worker(client=self.client) + self.federated = Federated(client=self.client, callback=callback) + self.scheduler = Scheduler(client=self.client, callback=callback) + self.worker = Worker(client=self.client, callback=callback) diff --git a/python/ofx/api/models/federated.py b/python/ofx/api/models/federated.py index d59d3c43c..d5f5dc356 100644 --- a/python/ofx/api/models/federated.py +++ b/python/ofx/api/models/federated.py @@ -16,9 +16,9 @@ class Federated(BaseAPI): - def create_job(self, job_id, roles, command_body): + def create_job(self, job_id, roles, initiator_party_id, command_body): return self.job_command(job_id=job_id, roles=roles, command="create", command_body=command_body, - parallel=False) + initiator_party_id=initiator_party_id, parallel=False) def stop_job(self, job_id, roles): return self.job_command(job_id=job_id, roles=roles, command="stop") diff --git a/python/ofx/api/models/resource.py b/python/ofx/api/models/resource.py index 704d4b501..6deca2d4a 100644 --- a/python/ofx/api/models/resource.py +++ b/python/ofx/api/models/resource.py @@ -51,6 +51,10 @@ def post(self, endpoint, data=None, json=None, **kwargs): return self.request('POST', url=self._set_url(endpoint), data=data, json=json, **self._set_request_timeout(kwargs)) + def send_file(self, endpoint, data=None, json=None, params=None, files=None, **kwargs): + return self.request('POST', url=self._set_url(endpoint), data=data, json=json, files=files, params=params, + **self._set_request_timeout(kwargs)) + def get(self, endpoint, **kwargs): kwargs.setdefault('allow_redirects', True) return self.request('GET', url=self._set_url(endpoint), **self._set_request_timeout(kwargs)) @@ -61,6 +65,10 @@ def put(self, endpoint, data=None, **kwargs): def delete(self, endpoint, **kwargs): return self.request('DELETE', url=self._set_url(endpoint), **self._set_request_timeout(kwargs)) + @property + def url(self): + return self._url + @property def _url(self): if self.version: @@ -81,7 +89,10 @@ def _set_request_timeout(self, kwargs): def _set_url(self, endpoint): return f"{self._url}/{endpoint}" - def remote(self, job_id, method, endpoint, src_party_id, dest_party_id, src_role, json_body, local=False, extra_params=None): + def remote(self, job_id, method, endpoint, src_party_id, dest_party_id, src_role, json_body, is_local=False, + extra_params=None, headers=None): + if not headers: + headers = {} if self.version: endpoint = f"/{self.version}{endpoint}" kwargs = { @@ -92,17 +103,20 @@ def remote(self, job_id, method, endpoint, src_party_id, dest_party_id, src_role 'dest_party_id': dest_party_id, 'src_role': src_role, 'json_body': json_body, - + "headers": headers } if extra_params: kwargs.update(extra_params) if not self.remote_host and not self.remote_port and self.remote_protocol == "grpc": - raise Exception(f'{self.remote_protocol} coordination communication protocol need remote host and remote port.') + raise Exception( + f'{self.remote_protocol} coordination communication protocol need remote host and remote port.') kwargs.update({ "source_host": self.host, "source_port": self.port, }) - if not local and self.remote_host and self.remote_port: + if is_local: + return self.remote_on_http(**kwargs) + if self.remote_host and self.remote_port: kwargs.update({ "host": self.remote_host, "port": self.remote_port, @@ -120,24 +134,27 @@ def remote(self, job_id, method, endpoint, src_party_id, dest_party_id, src_role return self.remote_on_http(**kwargs) def remote_on_http(self, method, endpoint, host=None, port=None, try_times=3, timeout=10, - json_body=None, dest_party_id=None, service_name="fateflow", **kwargs): + json_body=None, dest_party_id=None, service_name="fateflow", headers=None, **kwargs): + headers.update({ + "dest-party-id": dest_party_id, + "service": service_name + }) if host and port: url = f"{self.remote_protocol}://{host}:{port}{endpoint}" else: url = f"{self.base_url}{endpoint}" for t in range(try_times): try: - response = requests.request(method=method, url=url, timeout=timeout, json=json_body, headers={ - "dest-party-id": dest_party_id, - "service": service_name - }) + response = requests.request(method=method, url=url, timeout=timeout, json=json_body, headers=headers) response.raise_for_status() except Exception as e: if t >= try_times - 1: raise e else: - return response.json() - # time.sleep(get_exponential_backoff_interval(t)) + try: + return response.json() + except: + raise Exception(response.text) @staticmethod def remote_on_grpc_proxy(job_id, method, host, port, endpoint, src_party_id, dest_party_id, json_body, @@ -163,8 +180,10 @@ def remote_on_grpc_proxy(job_id, method, host, port, endpoint, src_party_id, des if t >= try_times - 1: raise e else: - return json.loads(bytes.decode(_return.body.value)) - # return json.loads(bytes.decode(_return.payload)) + try: + return json.loads(bytes.decode(_return.body.value)) + except Exception: + raise RuntimeError(f"{_return}, {_call}") finally: channel.close() @@ -191,18 +210,31 @@ def remote_on_grpc_osx(job_id, method, host, port, endpoint, src_party_id, dest_ if t >= try_times - 1: raise Exception(str(e)) else: - return json.loads(bytes.decode(_return.payload)) + try: + return json.loads(bytes.decode(_return.payload)) + except Exception: + raise RuntimeError(f"{_return}, {_call}") finally: channel.close() class BaseAPI: - def __init__(self, client: APIClient): + def __init__(self, client: APIClient, callback=None): self.client = client + self.callback = callback def federated_command(self, job_id, src_role, src_party_id, dest_role, dest_party_id, endpoint, body, - federated_response, method='POST', only_scheduler=False, extra_params=None): + federated_response, method='POST', only_scheduler=False, extra_params=None, + initiator_party_id=""): try: + headers = {} + if self.callback: + result = self.callback(dest_party_id, body, initiator_party_id=initiator_party_id) + if result.code == 0: + headers = result.signature if result.signature else {} + else: + raise Exception(result.code, result.message) + headers.update({"initiator_party_id": initiator_party_id}) response = self.client.remote(job_id=job_id, method=method, endpoint=endpoint, @@ -210,7 +242,9 @@ def federated_command(self, job_id, src_role, src_party_id, dest_role, dest_part src_party_id=src_party_id, dest_party_id=dest_party_id, json_body=body if body else {}, - extra_params=extra_params) + extra_params=extra_params, + is_local=self.is_local(party_id=dest_party_id), + headers=headers) if only_scheduler: return response except Exception as e: @@ -222,7 +256,11 @@ def federated_command(self, job_id, src_role, src_party_id, dest_role, dest_part return response federated_response[dest_role][dest_party_id] = response - def job_command(self, job_id, roles, command, command_body=None, parallel=False): + @staticmethod + def is_local(party_id): + return party_id == "0" + + def job_command(self, job_id, roles, command, command_body=None, parallel=False, initiator_party_id=""): federated_response = {} api_type = "partner/job" threads = [] @@ -238,12 +276,13 @@ def job_command(self, job_id, roles, command, command_body=None, parallel=False) command_body["party_id"] = dest_party_id command_body["job_id"] = job_id args = (job_id, "", "", dest_role, dest_party_id, endpoint, command_body, federated_response) + kwargs = {"initiator_party_id": initiator_party_id} if parallel: - t = threading.Thread(target=self.federated_command, args=args) + t = threading.Thread(target=self.federated_command, args=args, kwargs=kwargs) threads.append(t) t.start() else: - self.federated_command(*args) + self.federated_command(*args, initiator_party_id=initiator_party_id) for thread in threads: thread.join() return federated_response @@ -264,7 +303,8 @@ def task_command(self, tasks, command, command_body=None, parallel=False): dest_role, dest_party_id = task["role"], task["party_id"] federated_response[dest_role] = federated_response.get(dest_role, {}) endpoint = f"/partner/task/{command}" - args = (task['job_id'], task['role'], task['party_id'], dest_role, dest_party_id, endpoint, command_body, federated_response) + args = (task['job_id'], task['role'], task['party_id'], dest_role, dest_party_id, endpoint, command_body, + federated_response) if parallel: t = threading.Thread(target=self.federated_command, args=args) threads.append(t) @@ -275,7 +315,7 @@ def task_command(self, tasks, command, command_body=None, parallel=False): thread.join() return federated_response - def scheduler_command(self, command, party_id, command_body=None, method='POST'): + def scheduler_command(self, command, party_id, command_body=None, method='POST', initiator_party_id=""): try: federated_response = {} endpoint = f"/scheduler/{command}" @@ -288,7 +328,8 @@ def scheduler_command(self, command, party_id, command_body=None, method='POST') dest_party_id=party_id, body=command_body if command_body else {}, federated_response=federated_response, - only_scheduler=True + only_scheduler=True, + initiator_party_id=initiator_party_id ) except Exception as e: response = { diff --git a/python/ofx/api/models/scheduler.py b/python/ofx/api/models/scheduler.py index 843a09e70..0547c98b1 100644 --- a/python/ofx/api/models/scheduler.py +++ b/python/ofx/api/models/scheduler.py @@ -16,9 +16,10 @@ class Scheduler(BaseAPI): - def create_job(self, party_id, command_body): + def create_job(self, party_id, initiator_party_id, command_body): return self.scheduler_command(command="job/create", party_id=party_id, + initiator_party_id=initiator_party_id, command_body=command_body ) diff --git a/python/ofx/api/models/worker.py b/python/ofx/api/models/worker.py index 302ea41a4..fbd380686 100644 --- a/python/ofx/api/models/worker.py +++ b/python/ofx/api/models/worker.py @@ -16,31 +16,95 @@ class Worker(BaseAPI): + def report_task_status(self, status, execution_id, error=""): + if not error: + error = "" + endpoint = '/worker/task/status' + return self.client.post(endpoint=endpoint, json={ + "status": status, + "execution_id": execution_id, + "error": error + }) - def task_parameters(self, task_info): - endpoint = '/party/{}/{}/{}/{}/{}/{}/report'.format( - task_info["job_id"], - task_info["component_name"], - task_info["task_id"], - task_info["task_version"], - task_info["role"], - task_info["party_id"] + def query_task_status(self, execution_id): + endpoint = '/worker/task/status' + return self.client.get(endpoint=endpoint, json={ + "execution_id": execution_id, + }) + + def save_model(self, model_id, model_version, execution_id, output_key, type_name, fp): + files = {"file": fp} + return self.client.send_file( + endpoint="/worker/model/save", + files=files, + data={ + "model_id": model_id, + "model_version": model_version, + "execution_id": execution_id, + "output_key": output_key, + "type_name": type_name + }) + + def save_data_tracking(self, execution_id, output_key, meta_data, uri, namespace, name, overview, source, data_type, + index, partitions=None): + return self.client.post( + endpoint="/worker/data/tracking/save", + json={ + "execution_id": execution_id, + "output_key": output_key, + "meta_data": meta_data, + "uri": uri, + "namespace": namespace, + "name": name, + "overview": overview, + "source": source, + "data_type": data_type, + "index": index, + "partitions": partitions + }) + + def query_data_meta(self, job_id=None, role=None, party_id=None, task_name=None, output_key=None, namespace=None, + name=None): + # [job_id, role, party_id, task_name, output_key] or [name, namespace] + if namespace and name: + params = { + "namespace": namespace, + "name": name + } + else: + params = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "task_name": task_name, + "output_key": output_key + } + return self.client.get( + endpoint="/worker/data/tracking/query", + params=params ) - return self.client.post(endpoint=endpoint, json=task_info) - - def report_task(self, task_info): - endpoint = '/party/{}/{}/{}/{}/{}/{}/report'.format( - task_info["job_id"], - task_info["component_name"], - task_info["task_id"], - task_info["task_version"], - task_info["role"], - task_info["party_id"] + + def download_model(self, model_id, model_version, task_name, output_key, role, party_id): + return self.client.get( + endpoint="/worker/model/download", + params={ + "model_id": model_id, + "model_version": model_version, + "task_name": task_name, + "output_key": output_key, + "role": role, + "party_id": party_id + } ) - return self.client.post(endpoint=endpoint, json=task_info) - def output_metric(self, content): - return self.client.post(endpoint="/worker/metric/write", json=content) + def save_metric(self, execution_id, data): + return self.client.post( + endpoint="/worker/metric/save", + json={ + "execution_id": execution_id, + "data": data + }) - def write_model(self, content): - return self.client.post(endpoint="/worker/model/write", json=content) + def get_metric_save_url(self, execution_id): + endpoint = f"/worker/metric/save/{execution_id}" + return f"{self.client.url}{endpoint}" diff --git a/python/ofx/api/proto/__init__.py b/python/ofx/api/proto/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/ofx/api/proto/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/requirements-fate.txt b/python/requirements-fate.txt deleted file mode 100644 index 6b9d67900..000000000 --- a/python/requirements-fate.txt +++ /dev/null @@ -1,17 +0,0 @@ -click>=7.1.2,<8.0.0 -scikit-learn==1.0.1 -pandas==1.1.5 -protobuf==3.19.6 -pydantic -typing-extensions -ruamel-yaml==0.16.10 -requests<2.26.0 -cloudpickle==2.1.0 -lmdb==1.3.0 -numpy==1.23.1 -torch==1.13.1 -urllib3==1.26.5 -grpcio==1.46.3 -ml_metadata -beautifultable -rust_paillier diff --git a/python/requirements-flow.txt b/python/requirements-flow.txt index 10df577f7..de6a6a9c5 100644 --- a/python/requirements-flow.txt +++ b/python/requirements-flow.txt @@ -1,18 +1,24 @@ pip>=21 apsw<=3.10 -Flask==2.0.3 +Flask==2.2.5 grpcio==1.46.3 grpcio-tools==1.46.3 requests<2.26.0 urllib3==1.26.5 -ruamel-yaml==0.16.10 +ruamel-yaml==0.16 cachetools==3.0.0 filelock==3.3.1 -pydantic +pydantic==1.10.12 webargs peewee==3.9.3 python-dotenv==0.13.0 -ruamel-yaml==0.16.10 pyyaml==5.4.1 networkx -psutil>=5.7.0 \ No newline at end of file +psutil>=5.7.0 +casbin_peewee_adapter +casbin +pymysql +kazoo +shortuuid +cos-python-sdk-v5==1.9.10 +typing-extensions==4.5.0 \ No newline at end of file diff --git a/python/requirements-pulsar.txt b/python/requirements-pulsar.txt index 933764daf..03726430b 100644 --- a/python/requirements-pulsar.txt +++ b/python/requirements-pulsar.txt @@ -1 +1 @@ -pulsar-client==2.10.0 \ No newline at end of file +pulsar-client==2.10.2 \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt index f7bab6909..786714ebd 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -15,6 +15,3 @@ # spark -r requirements-spark.txt - -# fate --r requirements-fate.txt \ No newline at end of file diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 000000000..36d1a7417 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,109 @@ +import os +import shutil + +import fate_flow +from setuptools import find_packages, setup, Command + +packages = find_packages(".") +install_requires = [ + "apsw==3.38.5.post1", + "Flask==2.2.5", + "grpcio==1.46.3", + "grpcio-tools==1.46.3", + "requests<2.26.0", + "urllib3==1.26.5", + "cachetools", + "filelock", + "pydantic", + "webargs", + "peewee", + "python-dotenv", + "pyyaml", + "networkx", + "psutil>=5.7.0", + "casbin_peewee_adapter", + "casbin", + "pymysql", + "kazoo", + "shortuuid", + "cos-python-sdk-v5", + "typing-extensions", + "ruamel-yaml==0.16", +] +extras_require = { + "rabbitmq": ["pika==1.2.1"], + "pulsar": ["pulsar-client==2.10.2"], + "spark": ["pyspark"], + "eggroll": [ + "grpcio==1.46.3", + "grpcio-tools==1.46.3", + "numba==0.56.4", + "protobuf==3.19.6", + "pyarrow==6.0.1", + "mmh3==3.0.0", + "cachetools>=3.0.0", + "cloudpickle==2.1.0", + "psutil>=5.7.0", + ], + "all": ["fate_flow[rabbitmq,pulsar,spark,eggroll]"], +} + + +CONF_NAME = "conf" +PACKAGE_NAME = "fate_flow" +ENV_NAME = "fateflow.env" +HOME = os.path.abspath("../") +CONF_PATH = os.path.join(HOME, CONF_NAME) +PACKAGE_CONF_PATH = os.path.join(HOME, "python", "fate_flow", CONF_NAME) +ENV_PATH = os.path.join(HOME, ENV_NAME) +PACKAGE_ENV_PATH = os.path.join(HOME, "python", "fate_flow", ENV_NAME) + +readme_path = os.path.join(HOME, "README.md") + +entry_points = {"console_scripts": ["fate_flow = fate_flow.commands.server_cli:flow_server_cli"]} + +if os.path.exists(readme_path): + with open(readme_path, "r", encoding='utf-8') as f: + long_description = f.read() +else: + long_description = "fate flow" + + +class InstallCommand(Command): + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + if os.path.exists(PACKAGE_CONF_PATH): + shutil.rmtree(PACKAGE_CONF_PATH) + shutil.copytree(CONF_PATH, PACKAGE_CONF_PATH) + shutil.copyfile(ENV_PATH, PACKAGE_ENV_PATH) + + +setup( + name="fate_flow", + version=fate_flow.__version__, + keywords=["federated learning scheduler"], + author="FederatedAI", + author_email="contact@FedAI.org", + long_description_content_type="text/markdown", + long_description=long_description, + license="Apache-2.0 License", + url="https://fate.fedai.org/", + packages=packages, + install_requires=install_requires, + extras_require=extras_require, + package_data={ + "fate_flow": [f"{CONF_NAME}/*", ENV_NAME, "commands/*"] + }, + python_requires=">=3.8", + cmdclass={ + "pre_install": InstallCommand, + }, + entry_points=entry_points +)